diff --git a/bin.js b/bin.js index fe392e20..6e17a457 100755 --- a/bin.js +++ b/bin.js @@ -1,2 +1,2 @@ #!/usr/bin/env node -require("./dist/cli/bin.js"); +require("./dist/bin.js"); diff --git a/src/4.3_to_5.0/migrate_43_to_50.test.ts b/src/4.3_to_5.0/migrate_43_to_50.test.ts index 0f4a2a64..f3f098a5 100644 --- a/src/4.3_to_5.0/migrate_43_to_50.test.ts +++ b/src/4.3_to_5.0/migrate_43_to_50.test.ts @@ -73,6 +73,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; expect(migratedUIFolder).toMatchSnapshot(); @@ -88,6 +89,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; expect(migratedUIFolder).toMatchSnapshot(); @@ -106,6 +108,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; @@ -125,6 +128,7 @@ describe("migrate_43_to_50", () => { keysOfWidgetPluginsToRemove, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); // In the ActiveUI 4 folder, the file with id `0xb` represents a saved Page Filters widget. @@ -191,6 +195,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; @@ -257,6 +262,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; diff --git a/src/4.3_to_5.0/migrate_43_to_50.ts b/src/4.3_to_5.0/migrate_43_to_50.ts index ea94c8e0..7cf14e23 100644 --- a/src/4.3_to_5.0/migrate_43_to_50.ts +++ b/src/4.3_to_5.0/migrate_43_to_50.ts @@ -216,6 +216,7 @@ export async function migrate_43_to_50( doesReportIncludeStacks, shouldUpdateFiltersMdx, treeTableColumnWidth, + shouldMigrateCalculatedMeasures, }: { errorReport: ErrorReport; counters: OutcomeCounters; @@ -224,6 +225,7 @@ export async function migrate_43_to_50( doesReportIncludeStacks: boolean; shouldUpdateFiltersMdx: boolean; treeTableColumnWidth?: [number, number]; + shouldMigrateCalculatedMeasures: boolean; }, ): Promise { if (contentServer.children?.ui === undefined) { @@ -492,7 +494,7 @@ export async function migrate_43_to_50( migratedUIFolder.children = { ...migratedUIFolder.children, - ...(legacyPivotFolder + ...(legacyPivotFolder && shouldMigrateCalculatedMeasures ? { calculated_measures: await migrateCalculatedMeasures( legacyPivotFolder, diff --git a/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.test.ts b/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.test.ts index 64e0a4cd..d8d4980e 100644 --- a/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.test.ts +++ b/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.test.ts @@ -4,6 +4,9 @@ import { uiCalculatedMeasuresFolder } from "../__test_resources__/uiCalculatedMe import { ErrorReport, OutcomeCounters } from "../../migration.types"; import _cloneDeep from "lodash/cloneDeep"; import _fromPairs from "lodash/fromPairs"; +import { sandboxDataModel } from "@activeviam/data-model-5.1/dist/__test_resources__"; + +const dataModels = { sandbox: sandboxDataModel }; const contentServerForTests = _cloneDeep(contentServer); const errorReport: ErrorReport = {}; @@ -45,15 +48,15 @@ migrateSavedCalculatedMeasures({ counters, doesReportIncludeStacks: false, step: "5.0 to 5.1", + dataModels, }); describe("migrateSavedCalculatedMeasures", () => { - it("migrates the serialized definitions of all calculated measures created with ActiveUI 5.0 and used in a saved dashboard or saved widget, into ones that are natively supported by ActivePivot", () => { + it("migrates the serialized definitions of all calculated measures created with ActiveUI 5.0, into ones that are natively supported by ActivePivot", () => { // `uiCalculatedMeasuresFolder` contains 5 calculated measures. - // "Exp gamma sum" is not used in any saved widgets or dashboards, it is not migrated. - expect(counters.calculated_measures.success).toEqual(4); - expect(counters.calculated_measures.failed).toEqual(1); + expect(counters.calculated_measures.success).toEqual(5); + // "Exp gamma sum" is not used in any saved widget or dashboard. By default, it is associated with the first cube in the data model: `EquityDerivativesCube`. // "CM in 2 cubes" is used in both `EquityDerivativesCube` and `EquityDerivativesCubeDist`. // All others are only used in `EquityDerivativesCube`. // "Log pv.SUM" is inside a folder. @@ -67,6 +70,7 @@ describe("migrateSavedCalculatedMeasures", () => { "[Measures].[testo]", "[Measures].[new measure*]", "[Measures].[Distinct count city]", + "[Measures].[Exp gamma sum]", "[Measures].[Log pv.SUM]", "[Measures].[CM in 2 cubes]", "[Measures].[Test calculated measure]", @@ -127,21 +131,4 @@ describe("migrateSavedCalculatedMeasures", () => { contentServerForTests.children!.ui.children!.calculated_measures, ).toBeUndefined(); }); - - it("adds a warning to the `errorReport` that a calculated measure has not been migrated if it is not used in any saved widgets or dashboards", () => { - expect(errorReport).toStrictEqual({ - calculated_measures: { - "501": { - error: { - message: - 'Warning: Calculated measure "Exp gamma sum" was not migrated because it is not currently used in any saved widgets or dashboards.', - }, - folderId: ["a14"], - folderName: ["New folder"], - name: "Exp gamma sum", - step: "5.0 to 5.1", - }, - }, - }); - }); }); diff --git a/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.ts b/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.ts index dcee10ae..60622261 100644 --- a/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.ts +++ b/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.ts @@ -7,6 +7,7 @@ import { ErrorReport, OutcomeCounters } from "../../migration.types"; import { _getFilesAncestry } from "../../_getFilesAncestry"; import { _serializeError } from "../../_serializeError"; import { _getMetaData } from "../../_getMetaData"; +import { DataModel } from "@activeviam/activeui-sdk-5.1"; const contentServerWithEmptyPivotCalculatedMeasuresFolder = { entry: { @@ -72,6 +73,7 @@ export function migrateSavedCalculatedMeasures({ doesReportIncludeStacks, step, contentServerVersion, + dataModels, }: { contentServer: ContentRecord; measureToCubeMapping: { [measureName: string]: string[] }; @@ -80,6 +82,9 @@ export function migrateSavedCalculatedMeasures({ doesReportIncludeStacks: boolean; step: string; contentServerVersion?: string; + dataModels: { + [serverKey: string]: DataModel; + }; }): void { const legacyCalculatedMeasuresFolder = contentServer.children?.ui?.children?.calculated_measures; @@ -89,6 +94,10 @@ export function migrateSavedCalculatedMeasures({ return; } + const catalogs = Object.values(dataModels)[0].catalogs; + const cubes = Object.values(catalogs)[0].cubes; + const nameOfFirstCube = Object.keys(cubes)[0]; + // Make sure that the folder /pivot/entitlements/cm folder exists. _defaultsDeep( contentServer, @@ -104,26 +113,12 @@ export function migrateSavedCalculatedMeasures({ const measureName = _getMetaData(structure, folderId, id).name!; try { - const cubeNames = measureToCubeMapping[measureName]; - if (!cubeNames) { - // The calculated measure is not used in any saved widgets or dashboards. - // Do not migrate it. - counters.calculated_measures.failed++; - _addErrorToReport(errorReport, { - contentType: "calculated_measures", - folderId, - folderName, - fileErrorReport: { - error: { - message: `Warning: Calculated measure "${measureName}" was not migrated because it is not currently used in any saved widgets or dashboards.`, - }, - }, - fileId: id, - name: measureName, - step, - }); - return; - } + const cubeNames = measureToCubeMapping[ + measureName + ] ?? /** The saved calculated measure is not used in any widget. + * So it's impossible to infer the cube it is associated with. + * Default to the first cube in the data models. + */ [nameOfFirstCube]; const migratedContent = migrateSavedCalculatedMeasureContent( JSON.parse(record.entry.content), diff --git a/src/5.0_to_5.1/migrate_5.0_to_5.1.ts b/src/5.0_to_5.1/migrate_5.0_to_5.1.ts index 3e69a835..843c80c7 100644 --- a/src/5.0_to_5.1/migrate_5.0_to_5.1.ts +++ b/src/5.0_to_5.1/migrate_5.0_to_5.1.ts @@ -72,6 +72,7 @@ export const migrate_50_to_51: MigrationFunction = ( doesReportIncludeStacks, step: "5.0 to 5.1", contentServerVersion, + dataModels, }); migrateSavedFilters( diff --git a/src/cli/bin.ts b/src/bin.ts similarity index 76% rename from src/cli/bin.ts rename to src/bin.ts index c831942c..a3d0b2f2 100644 --- a/src/cli/bin.ts +++ b/src/bin.ts @@ -1,15 +1,19 @@ import yargs from "yargs"; -import { BehaviorOnError } from "../migration.types"; -import { migrateContentServer } from "./scripts/migrateContentServer"; -import { migrateNotebook } from "./scripts/migrateNotebook"; +import fs from "fs-extra"; +import path from "path"; +import { BehaviorOnError } from "./migration.types"; +import { migrateNotebook } from "./migrateNotebook"; import { convertFromVersion, convertToVersion, convertVersions, validFromVersions, validToVersions, -} from "./scripts/convertAtotiToAUIVersions"; -import { getTreeColumnWidthFromArgs } from "../getTreeColumnWidthFromArgs"; +} from "./convertAtotiToAUIVersions"; +import { getTreeColumnWidthFromArgs } from "./getTreeColumnWidthFromArgs"; +import { migrateContentServer } from "./migrateContentServer"; +import { DataModel, ContentRecord } from "@activeviam/activeui-sdk-5.1"; +import { logMigrationReport } from "./logMigrationReport"; const supportedFileExtension = ["JSON", "IPYNB"]; @@ -127,7 +131,7 @@ yargs }); args.implies("stack", "debug"); }, - ({ + async ({ inputPath, outputPath, serversPath, @@ -141,6 +145,7 @@ yargs onError, }) => { const doesReportIncludeStacks = stack; + const behaviorOnError = onError; const fileExtension = getFileExtension(inputPath); const { fromVersion: validFromVersion, toVersion: validToVersion } = @@ -149,32 +154,59 @@ yargs toVersion, }); + const fileToMigrate: ContentRecord = await fs.readJSON(inputPath); + + const servers: { + [serverKey: string]: { + dataModel: DataModel<"raw">; + url: string; + }; + } & { contentServerVersion?: string } = await fs.readJSON(serversPath); + if (fileExtension === "JSON") { - // Ensure that Atoti versions are not used as versions to migrate content server - migrateContentServer({ - inputPath, - outputPath, - serversPath, + const { counters, errorReport } = await migrateContentServer({ + contentServer: fileToMigrate, + servers, fromVersion: validFromVersion, toVersion: validToVersion, - removeWidgets, - debug, + keysOfWidgetPluginsToRemove: removeWidgets, doesReportIncludeStacks, - onError, + behaviorOnError: onError, treeTableColumnWidth: treeColumnWidth ? getTreeColumnWidthFromArgs(treeColumnWidth) : undefined, shouldUpdateFiltersMdx: updateFiltersMdx === undefined ? true : updateFiltersMdx, }); + + const { dir } = path.parse(outputPath); + await Promise.all([ + fs.writeJSON(outputPath, fileToMigrate, { spaces: 2 }), + logMigrationReport({ + counters, + errorReport, + debug, + doesReportIncludeStacks, + migrationOutputDirectory: dir, + behaviorOnError, + }), + ]); } else { - migrateNotebook({ - inputPath, - outputPath, - serversPath, - fromVersion: validFromVersion, - toVersion: validToVersion, - }); + const { numberOfMigratedWidgets, numberOfFailures } = + await migrateNotebook({ + notebook: fileToMigrate, + servers, + fromVersion: validFromVersion, + toVersion: validToVersion, + }); + + await fs.writeJSON(outputPath, fileToMigrate, { spaces: 2 }); + console.log( + `- Succesfully migrated ${numberOfMigratedWidgets} widget(s).`, + ); + if (numberOfFailures > 0) { + console.log(`- Failed to migrate ${numberOfFailures} widget(s)`); + } } }, ) diff --git a/src/cli/scripts/migrateContentServer.ts b/src/cli/scripts/migrateContentServer.ts deleted file mode 100644 index 65095506..00000000 --- a/src/cli/scripts/migrateContentServer.ts +++ /dev/null @@ -1,299 +0,0 @@ -import fs from "fs-extra"; -import path from "path"; -import _capitalize from "lodash/capitalize"; -import _fromPairs from "lodash/fromPairs"; -import _mapValues from "lodash/mapValues"; -import { ContentRecord, DataModel } from "@activeviam/activeui-sdk-5.1"; -import { getIndexedDataModel } from "@activeviam/data-model-5.1"; -import { - BehaviorOnError, - OutcomeCounters, - MigrationFunction, -} from "../../migration.types"; -import { getMigrateDashboards } from "../../getMigrateDashboards"; -import { getMigrateSavedWidgets } from "../../getMigrateSavedWidgets"; -import { getMigrateSavedFilters } from "../../getMigrateSavedFilters"; -import { migrate_43_to_50 } from "../../4.3_to_5.0"; -import { migrate_50_to_51 } from "../../5.0_to_5.1"; -import { getContent } from "../../getContent"; -import { - AtotiUIFromVersion, - AtotiUIToVersion, -} from "./convertAtotiToAUIVersions"; - -const migrationSteps: { - from: string; - to: string; - migrate: MigrationFunction; -}[] = [{ from: "5.0", to: "5.1", migrate: migrate_50_to_51 }]; - -const fromVersions = migrationSteps.map(({ from }) => from); -const toVersions = migrationSteps.map(({ to }) => to); - -const summaryMessages: { - [folderName: string]: { - [outcome: string]: - | string - | { [behaviorOnError in BehaviorOnError]: string }; - }; -} = { - dashboards: { - success: "were successfully migrated.", - partial: - "were partially migrated, but errors occurred in some of the widgets they contain. These widgets were copied as is into the migrated dashboards.", - failed: { - "keep-original": - "could not be migrated because errors occurred during their migration. Their original versions were copied as is into the migrated folder.", - "keep-last-successful-version": - "could not be migrated because errors occurred during their migration. The version obtained after the last successful migration step was copied as is into the migrated folder.", - "keep-going": - "had errors occurring during their migration. They were tentatively migrated to the desired version, but very likely have issues and won't work well in the UI.", - }, - removed: - "were cleaned up because they could not be found in the ui/dashboards/structure folder. They were already not visible in your version of ActiveUI.", - }, - filters: { - success: "were successfully migrated.", - failed: { - "keep-original": - "could not be migrated because errors occurred during their migration. Their original versions were copied as is into the migrated folder.", - "keep-last-successful-version": - "could not be migrated because errors occurred during their migration. The version obtained after the last successful migration step was copied as is into the migrated folder.", - "keep-going": - "had errors occurring during their migration. They were tentatively migrated to the desired version, but very likely have issues and won't work well in the UI.", - }, - removed: - "were cleaned up because they could not be found in the ui/filters/structure folder. They were already not visible in your version of ActiveUI.", - }, - widgets: { - success: "were successfully migrated.", - partial: "were migrated with warnings.", - removed: - "were cleaned up because they could not be found in the ui/widgets/structure folder or because their keys were passed in the --remove-widgets option.", - failed: { - "keep-original": - "could not be migrated because errors occurred during their migration. Their original versions were copied as is into the migrated folder.", - "keep-last-successful-version": - "could not be migrated because errors occurred during their migration. The version obtained after the last successful migration step was copied as is into the migrated folder.", - "keep-going": - "had errors occurring during their migration. They were tentatively migrated to the desired version, but very likely have issues and won't work well in the UI.", - }, - }, - folders: { - removed: - "were cleaned up because they could not be found in their structure folder. They were already not visible in your version of ActiveUI.", - }, - calculated_measures: { - success: "were successfully migrated.", - failed: - "could not be migrated because errors occurred during their migration.", - }, -}; - -export async function migrateContentServer({ - inputPath, - outputPath, - serversPath, - fromVersion, - toVersion, - removeWidgets: keysOfWidgetPluginsToRemove, - debug, - doesReportIncludeStacks, - onError: behaviorOnError, - treeTableColumnWidth, - shouldUpdateFiltersMdx, -}: { - inputPath: string; - outputPath: string; - serversPath: string; - fromVersion: AtotiUIFromVersion; - toVersion: AtotiUIToVersion; - removeWidgets: string[]; - debug: boolean; - doesReportIncludeStacks: boolean; - onError: BehaviorOnError; - treeTableColumnWidth?: [number, number]; - shouldUpdateFiltersMdx: boolean; -}): Promise { - const contentServer: ContentRecord = await fs.readJSON(inputPath); - - const originalDashboardsContent = getContent( - contentServer, - "dashboard", - fromVersion, - ); - const originalWidgetsContent = getContent( - contentServer, - "widget", - fromVersion, - ); - const originalFiltersContent = getContent( - contentServer, - "filter", - fromVersion, - ); - - const servers: { - [serverKey: string]: { - dataModel: DataModel<"raw">; - url: string; - }; - } & { contentServerVersion?: string } = await fs.readJSON(serversPath); - - const contentServerVersion = servers.contentServerVersion; - // Later, `servers` is expected to have a structure like `{ [serverKey: string]: { ... } }`. - // Several functions depending on this structure are applied to it: `_mapKeys`, `_findKey`, etc. - // Deleting the "contentServerVersion" attribute from it enables those functions still being applied to `servers` without breaking. - delete servers.contentServerVersion; - - const counters = _fromPairs( - ["dashboards", "widgets", "filters", "folders", "calculated_measures"].map( - (type) => [ - type, - { - success: 0, - partial: 0, - failed: 0, - removed: 0, - }, - ], - ), - // _fromPairs returns a Dictionary. - // In this case, the keys used correspond to the attributes of OutcomeCounters. - ) as OutcomeCounters; - const errorReport = {}; - - // Handle the special case of 4.3 to 5.0 separately, as: - // - the corresponding migration function has a different signature than all others - // - in particular, it is the only migration step with async logic - if (fromVersion === "4.3") { - await migrate_43_to_50(contentServer, { - errorReport, - counters, - servers, - keysOfWidgetPluginsToRemove, - doesReportIncludeStacks, - shouldUpdateFiltersMdx, - treeTableColumnWidth, - }); - } - - const fromVersionIndex = - fromVersion === "4.3" ? 0 : fromVersions.indexOf(fromVersion); - const toVersionIndex = toVersions.indexOf(toVersion); - - const dataModels = _mapValues(servers, ({ dataModel }) => - getIndexedDataModel(dataModel), - ); - - migrationSteps - .slice(fromVersionIndex, toVersionIndex + 1) - .forEach(({ migrate, from, to }) => { - const step = `${from} to ${to}`; - const migrateDashboards = getMigrateDashboards(contentServer, { - originalContent: originalDashboardsContent, - dataModels, - keysOfWidgetPluginsToRemove, - errorReport, - counters, - doesReportIncludeStacks, - behaviorOnError, - step, - }); - - const migrateSavedWidgets = getMigrateSavedWidgets(contentServer, { - originalContent: originalWidgetsContent, - dataModels, - keysOfWidgetPluginsToRemove, - errorReport, - counters, - doesReportIncludeStacks, - behaviorOnError, - step, - }); - - const migrateSavedFilters = getMigrateSavedFilters(contentServer, { - originalContent: originalFiltersContent, - dataModels, - errorReport, - counters, - doesReportIncludeStacks, - behaviorOnError, - step, - }); - - migrate(contentServer, { - migrateDashboards, - migrateSavedWidgets, - migrateSavedFilters, - dataModels, - keysOfWidgetPluginsToRemove, - errorReport, - counters, - doesReportIncludeStacks, - contentServerVersion, - }); - }); - - const { dir } = path.parse(outputPath); - - await fs.writeJSON(outputPath, contentServer, { spaces: 2 }); - - console.log("--------- END OF CONTENT MIGRATION ---------"); - - Object.entries(counters) - .filter(([, countersForFolder]) => - Object.values(countersForFolder).some((value) => value > 0), - ) - .forEach(([folderName, countersForFolder]) => { - console.log(`\n# ${_capitalize(folderName)}`); - Object.entries(countersForFolder).forEach(([outcome, counter]) => { - if (counter > 0) { - console.log( - `- ${counter} ${ - outcome === "failed" && folderName !== "calculated_measures" - ? // Apart from calculated measures, all the content types with a failed outcome have a message per behavior on error. - ( - summaryMessages[folderName][outcome] as { - [behaviorOnError: string]: string; - } - )[behaviorOnError] - : summaryMessages[folderName][outcome] - }`, - ); - } - }); - }); - - console.log("\n"); - - if ( - counters.dashboards.failed + - counters.dashboards.partial + - counters.filters.failed + - counters.widgets.failed > - 0 - ) { - if (!debug) { - console.log(`For more information about the errors that occurred, rerun the command with the \`--debug\` option. -This will output a file named \`report.json\` containing the error messages.`); - } else { - console.log( - `See report.json for more information about the errors that occurred.`, - ); - } - if (!doesReportIncludeStacks) { - console.log( - "To see the stack traces of the errors in this file, you can also use the `--stack` option.", - ); - } - } - - console.log("--------------------------------------------"); - - if (errorReport && debug) { - await fs.writeJSON(path.join(...dir, "report.json"), errorReport, { - spaces: 2, - }); - } -} diff --git a/src/cli/scripts/convertAtotiToAUIVersions.test.ts b/src/convertAtotiToAUIVersions.test.ts similarity index 100% rename from src/cli/scripts/convertAtotiToAUIVersions.test.ts rename to src/convertAtotiToAUIVersions.test.ts diff --git a/src/cli/scripts/convertAtotiToAUIVersions.ts b/src/convertAtotiToAUIVersions.ts similarity index 100% rename from src/cli/scripts/convertAtotiToAUIVersions.ts rename to src/convertAtotiToAUIVersions.ts diff --git a/src/logMigrationReport.ts b/src/logMigrationReport.ts new file mode 100644 index 00000000..6b815d0c --- /dev/null +++ b/src/logMigrationReport.ts @@ -0,0 +1,151 @@ +import fs from "fs-extra"; +import path from "path"; +import _capitalize from "lodash/capitalize"; + +import { + BehaviorOnError, + ErrorReport, + OutcomeCounters, +} from "./migration.types"; + +const summaryMessages: { + [folderName: string]: { + [outcome: string]: + | string + | { [behaviorOnError in BehaviorOnError]: string }; + }; +} = { + dashboards: { + success: "were successfully migrated.", + partial: + "were partially migrated, but errors occurred in some of the widgets they contain. These widgets were copied as is into the migrated dashboards.", + failed: { + "keep-original": + "could not be migrated because errors occurred during their migration. Their original versions were copied as is into the migrated folder.", + "keep-last-successful-version": + "could not be migrated because errors occurred during their migration. The version obtained after the last successful migration step was copied as is into the migrated folder.", + "keep-going": + "had errors occurring during their migration. They were tentatively migrated to the desired version, but very likely have issues and won't work well in the UI.", + }, + removed: + "were cleaned up because they could not be found in the ui/dashboards/structure folder. They were already not visible in your version of ActiveUI.", + }, + filters: { + success: "were successfully migrated.", + failed: { + "keep-original": + "could not be migrated because errors occurred during their migration. Their original versions were copied as is into the migrated folder.", + "keep-last-successful-version": + "could not be migrated because errors occurred during their migration. The version obtained after the last successful migration step was copied as is into the migrated folder.", + "keep-going": + "had errors occurring during their migration. They were tentatively migrated to the desired version, but very likely have issues and won't work well in the UI.", + }, + removed: + "were cleaned up because they could not be found in the ui/filters/structure folder. They were already not visible in your version of ActiveUI.", + }, + widgets: { + success: "were successfully migrated.", + partial: "were migrated with warnings.", + removed: + "were cleaned up because they could not be found in the ui/widgets/structure folder or because their keys were passed in the --remove-widgets option.", + failed: { + "keep-original": + "could not be migrated because errors occurred during their migration. Their original versions were copied as is into the migrated folder.", + "keep-last-successful-version": + "could not be migrated because errors occurred during their migration. The version obtained after the last successful migration step was copied as is into the migrated folder.", + "keep-going": + "had errors occurring during their migration. They were tentatively migrated to the desired version, but very likely have issues and won't work well in the UI.", + }, + }, + folders: { + removed: + "were cleaned up because they could not be found in their structure folder. They were already not visible in your version of ActiveUI.", + }, + calculated_measures: { + success: "were successfully migrated.", + failed: + "could not be migrated because errors occurred during their migration.", + }, +}; + +/** + * Logs the outcome of the migration in the console, which includes the number of successful/failed individual resource migrations. + * If `debug` is set to `true`, also logs the detailed `errorReport` in a `report.json` file under `migrationOutputDirectory`. + */ +export async function logMigrationReport({ + counters, + errorReport, + debug, + doesReportIncludeStacks, + behaviorOnError, + migrationOutputDirectory, +}: { + counters: OutcomeCounters; + errorReport: ErrorReport; + debug: boolean; + doesReportIncludeStacks: boolean; + behaviorOnError: BehaviorOnError; + migrationOutputDirectory: string; +}): Promise { + console.log("--------- END OF CONTENT MIGRATION ---------"); + + Object.entries(counters) + .filter(([, countersForFolder]) => + Object.values(countersForFolder).some((value) => value > 0), + ) + .forEach(([folderName, countersForFolder]) => { + console.log(`\n# ${_capitalize(folderName)}`); + Object.entries(countersForFolder).forEach(([outcome, counter]) => { + if (counter > 0) { + console.log( + `- ${counter} ${ + outcome === "failed" && folderName !== "calculated_measures" + ? // Apart from calculated measures, all the content types with a failed outcome have a message per behavior on error. + ( + summaryMessages[folderName][outcome] as { + [behaviorOnError: string]: string; + } + )[behaviorOnError] + : summaryMessages[folderName][outcome] + }`, + ); + } + }); + }); + + console.log("\n"); + + if ( + counters.dashboards.failed + + counters.dashboards.partial + + counters.filters.failed + + counters.widgets.failed > + 0 + ) { + if (!debug) { + console.log(`For more information about the errors that occurred, rerun the command with the \`--debug\` option. + This will output a file named \`report.json\` containing the error messages.`); + } else { + console.log( + `See report.json for more information about the errors that occurred.`, + ); + } + if (!doesReportIncludeStacks) { + console.log( + "To see the stack traces of the errors in this file, you can also use the `--stack` option.", + ); + } + } + + console.log("--------------------------------------------"); + + if (errorReport && debug) { + await fs.writeJSON( + path.join(...migrationOutputDirectory, "report.json"), + errorReport, + { + spaces: 2, + }, + ); + } +} diff --git a/src/migrateContentServer.test.ts b/src/migrateContentServer.test.ts new file mode 100644 index 00000000..c0667927 --- /dev/null +++ b/src/migrateContentServer.test.ts @@ -0,0 +1,235 @@ +import { ContentRecord } from "@activeviam/activeui-sdk-5.1"; +import { servers } from "./4.3_to_5.0/__test_resources__/servers"; +import { smallLegacyPivotFolder } from "./4.3_to_5.0/__test_resources__/smallLegacyPivotFolder"; +import { smallLegacyUIFolder } from "./4.3_to_5.0/__test_resources__/smallLegacyUIFolder"; +import { migrateContentServer } from "./migrateContentServer"; +import _cloneDeep from "lodash/cloneDeep"; + +jest.mock(`./4.3_to_5.0/generateId`, () => { + let counter = 0; + return { + generateId: jest.fn(() => { + const id = `00${counter}`; + counter += 1; + + return id; + }), + }; +}); + +describe("migrateContentServer", () => { + it("migrates calculated measures from the /pivot folder to the /ui folder when migrating from 4.3 to 5.0", async () => { + const contentServer: ContentRecord = { + children: { ui: smallLegacyUIFolder, pivot: smallLegacyPivotFolder }, + entry: { + owners: [], + readers: [], + isDirectory: true, + canRead: true, + canWrite: false, + lastEditor: "Freddie Mercury", + timestamp: 0xbeef, + }, + }; + + await migrateContentServer({ + contentServer, + servers, + fromVersion: "4.3", + toVersion: "5.0", + keysOfWidgetPluginsToRemove: [], + doesReportIncludeStacks: false, + shouldUpdateFiltersMdx: true, + behaviorOnError: "keep-original", + }); + + expect(contentServer.children?.ui.children?.calculated_measures) + .toMatchInlineSnapshot(` + { + "children": { + "content": { + "children": { + "000": { + "entry": { + "content": "{"expression":"IIf(IsEmpty(([Booking].[Desk].CurrentMember.Parent, [Measures].[pnl.SUM])), null, [Measures].[pnl.SUM] / ([Booking].[Desk].CurrentMember.Parent, [Measures].[pnl.SUM]))","properties":["FORMAT_STRING=\\"#,###.##%\\""]}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + "001": { + "entry": { + "content": "{"expression":"IIf(IsEmpty(([Currency].[Currency].[ALL].[AllMember], [Measures].[pnl.FOREX])), null, [Measures].[pnl.FOREX] / ([Currency].[Currency].[ALL].[AllMember], [Measures].[pnl.FOREX]))","properties":["FORMAT_STRING=\\"#,###.##%\\""]}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + "002": { + "entry": { + "content": "{"expression":"Count(Descendants([CounterParty].[CounterParty].CurrentMember, [CounterParty].[CounterParty].[CounterPartyGroup]), EXCLUDEEMPTY)","properties":["FORMAT_STRING=\\"#,###.##\\""]}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "ROLE_USER", + ], + "readers": [ + "ROLE_USER", + ], + }, + }, + "structure": { + "children": { + "000": { + "children": { + "000_metadata": { + "entry": { + "content": "{"name":"[Measures].[third calculated measure]"}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + "001": { + "children": { + "001_metadata": { + "entry": { + "content": "{"name":"[Measures].[second calculated measure]"}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + "002": { + "children": { + "002_metadata": { + "entry": { + "content": "{"name":"[Measures].[first calculated measure]"}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "ROLE_USER", + ], + "readers": [ + "ROLE_USER", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "ROLE_CS_ROOT", + ], + "readers": [ + "ROLE_CS_ROOT", + ], + }, + } + `); + }); + + it("does not migrate calculated measures when migrating from 4.3 to > 5.0", async () => { + const contentServer: ContentRecord = { + children: { ui: smallLegacyUIFolder, pivot: smallLegacyPivotFolder }, + entry: { + owners: [], + readers: [], + isDirectory: true, + canRead: true, + canWrite: false, + lastEditor: "Freddie Mercury", + timestamp: 0xbeef, + }, + }; + + const contentServerBeforeMigration = _cloneDeep(contentServer); + + await migrateContentServer({ + contentServer, + servers, + fromVersion: "4.3", + toVersion: "5.1", + keysOfWidgetPluginsToRemove: [], + doesReportIncludeStacks: false, + shouldUpdateFiltersMdx: true, + behaviorOnError: "keep-original", + }); + + expect(contentServer.children?.pivot).toStrictEqual( + contentServerBeforeMigration.children?.pivot, + ); + expect(contentServer.children?.ui.children?.calculated_measures).toBe( + undefined, + ); + }); +}); diff --git a/src/migrateContentServer.ts b/src/migrateContentServer.ts new file mode 100644 index 00000000..afb63f53 --- /dev/null +++ b/src/migrateContentServer.ts @@ -0,0 +1,181 @@ +import _fromPairs from "lodash/fromPairs"; +import _mapValues from "lodash/mapValues"; +import { ContentRecord } from "@activeviam/activeui-sdk-5.1"; +import { getIndexedDataModel, DataModel } from "@activeviam/data-model-5.1"; +import { + BehaviorOnError, + OutcomeCounters, + MigrationFunction, +} from "./migration.types"; +import { getMigrateDashboards } from "./getMigrateDashboards"; +import { getMigrateSavedWidgets } from "./getMigrateSavedWidgets"; +import { getMigrateSavedFilters } from "./getMigrateSavedFilters"; +import { migrate_43_to_50 } from "./4.3_to_5.0"; +import { migrate_50_to_51 } from "./5.0_to_5.1"; +import { getContent } from "./getContent"; +import { + AtotiUIFromVersion, + AtotiUIToVersion, +} from "./convertAtotiToAUIVersions"; + +const migrationSteps: { + from: string; + to: string; + migrate: MigrationFunction; +}[] = [{ from: "5.0", to: "5.1", migrate: migrate_50_to_51 }]; + +const fromVersions = migrationSteps.map(({ from }) => from); +const toVersions = migrationSteps.map(({ to }) => to); + +/** + * Migrates `contentServer` from an older version of Atoti UI to a more recent one. + * Also keeps track of the number of migration successes and failures in `counters` and a detailed `errorReport`. + * + * Mutates `contentServer`, `errorReport` and `counters`. + */ +export async function migrateContentServer({ + contentServer, + servers, + fromVersion, + toVersion, + keysOfWidgetPluginsToRemove, + doesReportIncludeStacks, + behaviorOnError, + treeTableColumnWidth, + shouldUpdateFiltersMdx, +}: { + contentServer: ContentRecord; + servers: { + [serverKey: string]: { + dataModel: DataModel<"raw">; + url: string; + }; + } & { contentServerVersion?: string }; + fromVersion: AtotiUIFromVersion; + toVersion: AtotiUIToVersion; + keysOfWidgetPluginsToRemove: string[]; + doesReportIncludeStacks: boolean; + behaviorOnError: BehaviorOnError; + treeTableColumnWidth?: [number, number]; + shouldUpdateFiltersMdx: boolean; +}): Promise<{ + counters: OutcomeCounters; + errorReport: Record; +}> { + const originalDashboardsContent = getContent( + contentServer, + "dashboard", + fromVersion, + ); + const originalWidgetsContent = getContent( + contentServer, + "widget", + fromVersion, + ); + const originalFiltersContent = getContent( + contentServer, + "filter", + fromVersion, + ); + + const contentServerVersion = servers.contentServerVersion; + // Later, `servers` is expected to have a structure like `{ [serverKey: string]: { ... } }`. + // Several functions depending on this structure are applied to it: `_mapKeys`, `_findKey`, etc. + // Deleting the "contentServerVersion" attribute from it enables those functions still being applied to `servers` without breaking. + delete servers.contentServerVersion; + + const counters = _fromPairs( + ["dashboards", "widgets", "filters", "folders", "calculated_measures"].map( + (type) => [ + type, + { + success: 0, + partial: 0, + failed: 0, + removed: 0, + }, + ], + ), + // _fromPairs returns a Dictionary. + // In this case, the keys used correspond to the attributes of OutcomeCounters. + ) as OutcomeCounters; + const errorReport = {}; + + // Handle the special case of 4.3 to 5.0 separately, as: + // - the corresponding migration function has a different signature than all others + // - in particular, it is the only migration step with async logic + if (fromVersion === "4.3") { + await migrate_43_to_50(contentServer, { + errorReport, + counters, + servers, + keysOfWidgetPluginsToRemove, + doesReportIncludeStacks, + shouldUpdateFiltersMdx, + treeTableColumnWidth, + // In Atoti UI 4.3 and 5.1+, saved calculated measures are an Atoti Server concept, and are stored under /pivot in the content server. + // In Atoti UI 5.0, saved calculated measures are an Atoti UI concept, and are stored under /ui in the content server. + // So it's only when the target version is 5.0 that calculated measures need to be migrated. + shouldMigrateCalculatedMeasures: toVersion === "5.0", + }); + } + + const fromVersionIndex = + fromVersion === "4.3" ? 0 : fromVersions.indexOf(fromVersion); + const toVersionIndex = toVersions.indexOf(toVersion); + + const dataModels = _mapValues(servers, ({ dataModel }) => + getIndexedDataModel(dataModel), + ); + + migrationSteps + .slice(fromVersionIndex, toVersionIndex + 1) + .forEach(({ migrate, from, to }) => { + const step = `${from} to ${to}`; + const migrateDashboards = getMigrateDashboards(contentServer, { + originalContent: originalDashboardsContent, + dataModels, + keysOfWidgetPluginsToRemove, + errorReport, + counters, + doesReportIncludeStacks, + behaviorOnError, + step, + }); + + const migrateSavedWidgets = getMigrateSavedWidgets(contentServer, { + originalContent: originalWidgetsContent, + dataModels, + keysOfWidgetPluginsToRemove, + errorReport, + counters, + doesReportIncludeStacks, + behaviorOnError, + step, + }); + + const migrateSavedFilters = getMigrateSavedFilters(contentServer, { + originalContent: originalFiltersContent, + dataModels, + errorReport, + counters, + doesReportIncludeStacks, + behaviorOnError, + step, + }); + + migrate(contentServer, { + migrateDashboards, + migrateSavedWidgets, + migrateSavedFilters, + dataModels, + keysOfWidgetPluginsToRemove, + errorReport, + counters, + doesReportIncludeStacks, + contentServerVersion, + }); + }); + + return { errorReport, counters }; +} diff --git a/src/cli/scripts/migrateNotebook.ts b/src/migrateNotebook.ts similarity index 76% rename from src/cli/scripts/migrateNotebook.ts rename to src/migrateNotebook.ts index 7fdc66e2..2de1d520 100644 --- a/src/cli/scripts/migrateNotebook.ts +++ b/src/migrateNotebook.ts @@ -1,10 +1,9 @@ -import { migrateWidget } from "../../5.0_to_5.1/migrateWidget"; -import fs from "fs-extra"; +import { migrateWidget } from "./5.0_to_5.1/migrateWidget"; import { DataModel, getIndexedDataModel } from "@activeviam/data-model-5.1"; import _mapValues from "lodash/mapValues"; import { serializeWidgetState } from "@activeviam/activeui-sdk-5.1"; import { deserializeWidgetState } from "@activeviam/activeui-sdk-5.0"; -import { MigrateWidgetCallback } from "../../migration.types"; +import { MigrateWidgetCallback } from "./migration.types"; import { AtotiUIFromVersion, AtotiUIToVersion, @@ -21,24 +20,18 @@ const migrationSteps: { * Migrates the Atoti UI widgets from 5.0 to 5.1 of an Atoti Jupter Notebook. */ export const migrateNotebook = async ({ - inputPath, - outputPath, - serversPath, + notebook, + servers, fromVersion, toVersion, }: { - inputPath: string; - outputPath: string; - serversPath: string; + notebook: any; + servers: { + [serverKey: string]: { dataModel: DataModel<"raw">; url: string }; + }; fromVersion: AtotiUIFromVersion; toVersion: AtotiUIToVersion; -}): Promise => { - const notebook = await fs.readJSON(inputPath); - - const servers: { - [serverKey: string]: { dataModel: DataModel<"raw">; url: string }; - } = await fs.readJSON(serversPath); - +}): Promise<{ numberOfMigratedWidgets: number; numberOfFailures: number }> => { const dataModels = _mapValues(servers, ({ dataModel }) => getIndexedDataModel(dataModel), ); @@ -77,10 +70,5 @@ export const migrateNotebook = async ({ } } - await fs.writeJSON(outputPath, notebook, { spaces: 2 }); - - console.log(`- Succesfully migrated ${numberOfMigratedWidgets} widget(s).`); - if (numberOfFailures > 0) { - console.log(`- Failed to migrate ${numberOfFailures} widget(s)`); - } + return { numberOfMigratedWidgets, numberOfFailures }; }; diff --git a/webpack.config.js b/webpack.config.js index c0373390..443cd44e 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -7,7 +7,7 @@ const webpack = require("webpack"); // This allows the functions exported by this package to be run smoothly in a Node.js environment (see https://support.activeviam.com/jira/browse/UI-6165). module.exports = { entry: { - bin: "./src/cli/bin.ts", + bin: "./src/bin.ts", }, module: { rules: [ @@ -39,8 +39,8 @@ module.exports = { output: { hashFunction: "xxhash64", globalObject: "window", - filename: (pathData) => { - return pathData.chunk.name === "bin" ? "cli/[name].js" : "[name].js"; + filename: () => { + return "[name].js"; }, path: path.resolve(__dirname, "dist"), },