From 17b46764f00b8387d4184b8346ac2451b1b4d953 Mon Sep 17 00:00:00 2001 From: Antoine Tissier Date: Thu, 16 May 2024 11:36:00 +0200 Subject: [PATCH 1/4] UI-9435 - Better handling of calculated measures --- src/4.3_to_5.0/migrate_43_to_50.test.ts | 6 + src/4.3_to_5.0/migrate_43_to_50.ts | 4 +- .../migrateSavedCalculatedMeasures.test.ts | 29 +-- .../migrateSavedCalculatedMeasures.ts | 35 ++- src/5.0_to_5.1/migrate_5.0_to_5.1.ts | 1 + src/cli/bin.ts | 6 +- ...tServer.ts => migrateContentServerJson.ts} | 153 ++---------- src/cli/scripts/migrateNotebook.ts | 2 +- .../convertAtotiToAUIVersions.test.ts | 0 .../scripts => }/convertAtotiToAUIVersions.ts | 0 src/migrateContentServer.test.ts | 234 ++++++++++++++++++ src/migrateContentServer.ts | 181 ++++++++++++++ 12 files changed, 470 insertions(+), 181 deletions(-) rename src/cli/scripts/{migrateContentServer.ts => migrateContentServerJson.ts} (60%) rename src/{cli/scripts => }/convertAtotiToAUIVersions.test.ts (100%) rename src/{cli/scripts => }/convertAtotiToAUIVersions.ts (100%) create mode 100644 src/migrateContentServer.test.ts create mode 100644 src/migrateContentServer.ts diff --git a/src/4.3_to_5.0/migrate_43_to_50.test.ts b/src/4.3_to_5.0/migrate_43_to_50.test.ts index 0f4a2a64..f3f098a5 100644 --- a/src/4.3_to_5.0/migrate_43_to_50.test.ts +++ b/src/4.3_to_5.0/migrate_43_to_50.test.ts @@ -73,6 +73,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; expect(migratedUIFolder).toMatchSnapshot(); @@ -88,6 +89,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; expect(migratedUIFolder).toMatchSnapshot(); @@ -106,6 +108,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; @@ -125,6 +128,7 @@ describe("migrate_43_to_50", () => { keysOfWidgetPluginsToRemove, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); // In the ActiveUI 4 folder, the file with id `0xb` represents a saved Page Filters widget. @@ -191,6 +195,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; @@ -257,6 +262,7 @@ describe("migrate_43_to_50", () => { servers, doesReportIncludeStacks: false, shouldUpdateFiltersMdx: true, + shouldMigrateCalculatedMeasures: true, }); const migratedUIFolder = contentServer.children?.ui; diff --git a/src/4.3_to_5.0/migrate_43_to_50.ts b/src/4.3_to_5.0/migrate_43_to_50.ts index ea94c8e0..7cf14e23 100644 --- a/src/4.3_to_5.0/migrate_43_to_50.ts +++ b/src/4.3_to_5.0/migrate_43_to_50.ts @@ -216,6 +216,7 @@ export async function migrate_43_to_50( doesReportIncludeStacks, shouldUpdateFiltersMdx, treeTableColumnWidth, + shouldMigrateCalculatedMeasures, }: { errorReport: ErrorReport; counters: OutcomeCounters; @@ -224,6 +225,7 @@ export async function migrate_43_to_50( doesReportIncludeStacks: boolean; shouldUpdateFiltersMdx: boolean; treeTableColumnWidth?: [number, number]; + shouldMigrateCalculatedMeasures: boolean; }, ): Promise { if (contentServer.children?.ui === undefined) { @@ -492,7 +494,7 @@ export async function migrate_43_to_50( migratedUIFolder.children = { ...migratedUIFolder.children, - ...(legacyPivotFolder + ...(legacyPivotFolder && shouldMigrateCalculatedMeasures ? { calculated_measures: await migrateCalculatedMeasures( legacyPivotFolder, diff --git a/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.test.ts b/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.test.ts index 64e0a4cd..d8d4980e 100644 --- a/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.test.ts +++ b/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.test.ts @@ -4,6 +4,9 @@ import { uiCalculatedMeasuresFolder } from "../__test_resources__/uiCalculatedMe import { ErrorReport, OutcomeCounters } from "../../migration.types"; import _cloneDeep from "lodash/cloneDeep"; import _fromPairs from "lodash/fromPairs"; +import { sandboxDataModel } from "@activeviam/data-model-5.1/dist/__test_resources__"; + +const dataModels = { sandbox: sandboxDataModel }; const contentServerForTests = _cloneDeep(contentServer); const errorReport: ErrorReport = {}; @@ -45,15 +48,15 @@ migrateSavedCalculatedMeasures({ counters, doesReportIncludeStacks: false, step: "5.0 to 5.1", + dataModels, }); describe("migrateSavedCalculatedMeasures", () => { - it("migrates the serialized definitions of all calculated measures created with ActiveUI 5.0 and used in a saved dashboard or saved widget, into ones that are natively supported by ActivePivot", () => { + it("migrates the serialized definitions of all calculated measures created with ActiveUI 5.0, into ones that are natively supported by ActivePivot", () => { // `uiCalculatedMeasuresFolder` contains 5 calculated measures. - // "Exp gamma sum" is not used in any saved widgets or dashboards, it is not migrated. - expect(counters.calculated_measures.success).toEqual(4); - expect(counters.calculated_measures.failed).toEqual(1); + expect(counters.calculated_measures.success).toEqual(5); + // "Exp gamma sum" is not used in any saved widget or dashboard. By default, it is associated with the first cube in the data model: `EquityDerivativesCube`. // "CM in 2 cubes" is used in both `EquityDerivativesCube` and `EquityDerivativesCubeDist`. // All others are only used in `EquityDerivativesCube`. // "Log pv.SUM" is inside a folder. @@ -67,6 +70,7 @@ describe("migrateSavedCalculatedMeasures", () => { "[Measures].[testo]", "[Measures].[new measure*]", "[Measures].[Distinct count city]", + "[Measures].[Exp gamma sum]", "[Measures].[Log pv.SUM]", "[Measures].[CM in 2 cubes]", "[Measures].[Test calculated measure]", @@ -127,21 +131,4 @@ describe("migrateSavedCalculatedMeasures", () => { contentServerForTests.children!.ui.children!.calculated_measures, ).toBeUndefined(); }); - - it("adds a warning to the `errorReport` that a calculated measure has not been migrated if it is not used in any saved widgets or dashboards", () => { - expect(errorReport).toStrictEqual({ - calculated_measures: { - "501": { - error: { - message: - 'Warning: Calculated measure "Exp gamma sum" was not migrated because it is not currently used in any saved widgets or dashboards.', - }, - folderId: ["a14"], - folderName: ["New folder"], - name: "Exp gamma sum", - step: "5.0 to 5.1", - }, - }, - }); - }); }); diff --git a/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.ts b/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.ts index dcee10ae..60622261 100644 --- a/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.ts +++ b/src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.ts @@ -7,6 +7,7 @@ import { ErrorReport, OutcomeCounters } from "../../migration.types"; import { _getFilesAncestry } from "../../_getFilesAncestry"; import { _serializeError } from "../../_serializeError"; import { _getMetaData } from "../../_getMetaData"; +import { DataModel } from "@activeviam/activeui-sdk-5.1"; const contentServerWithEmptyPivotCalculatedMeasuresFolder = { entry: { @@ -72,6 +73,7 @@ export function migrateSavedCalculatedMeasures({ doesReportIncludeStacks, step, contentServerVersion, + dataModels, }: { contentServer: ContentRecord; measureToCubeMapping: { [measureName: string]: string[] }; @@ -80,6 +82,9 @@ export function migrateSavedCalculatedMeasures({ doesReportIncludeStacks: boolean; step: string; contentServerVersion?: string; + dataModels: { + [serverKey: string]: DataModel; + }; }): void { const legacyCalculatedMeasuresFolder = contentServer.children?.ui?.children?.calculated_measures; @@ -89,6 +94,10 @@ export function migrateSavedCalculatedMeasures({ return; } + const catalogs = Object.values(dataModels)[0].catalogs; + const cubes = Object.values(catalogs)[0].cubes; + const nameOfFirstCube = Object.keys(cubes)[0]; + // Make sure that the folder /pivot/entitlements/cm folder exists. _defaultsDeep( contentServer, @@ -104,26 +113,12 @@ export function migrateSavedCalculatedMeasures({ const measureName = _getMetaData(structure, folderId, id).name!; try { - const cubeNames = measureToCubeMapping[measureName]; - if (!cubeNames) { - // The calculated measure is not used in any saved widgets or dashboards. - // Do not migrate it. - counters.calculated_measures.failed++; - _addErrorToReport(errorReport, { - contentType: "calculated_measures", - folderId, - folderName, - fileErrorReport: { - error: { - message: `Warning: Calculated measure "${measureName}" was not migrated because it is not currently used in any saved widgets or dashboards.`, - }, - }, - fileId: id, - name: measureName, - step, - }); - return; - } + const cubeNames = measureToCubeMapping[ + measureName + ] ?? /** The saved calculated measure is not used in any widget. + * So it's impossible to infer the cube it is associated with. + * Default to the first cube in the data models. + */ [nameOfFirstCube]; const migratedContent = migrateSavedCalculatedMeasureContent( JSON.parse(record.entry.content), diff --git a/src/5.0_to_5.1/migrate_5.0_to_5.1.ts b/src/5.0_to_5.1/migrate_5.0_to_5.1.ts index 3e69a835..6d945a2d 100644 --- a/src/5.0_to_5.1/migrate_5.0_to_5.1.ts +++ b/src/5.0_to_5.1/migrate_5.0_to_5.1.ts @@ -72,6 +72,7 @@ export const migrate_50_to_51: MigrationFunction = ( doesReportIncludeStacks, step: "5.0 to 5.1", contentServerVersion, + dataModels }); migrateSavedFilters( diff --git a/src/cli/bin.ts b/src/cli/bin.ts index c831942c..6f5d0611 100644 --- a/src/cli/bin.ts +++ b/src/cli/bin.ts @@ -1,6 +1,6 @@ import yargs from "yargs"; import { BehaviorOnError } from "../migration.types"; -import { migrateContentServer } from "./scripts/migrateContentServer"; +import { migrateContentServerJson } from "./scripts/migrateContentServerJson"; import { migrateNotebook } from "./scripts/migrateNotebook"; import { convertFromVersion, @@ -8,7 +8,7 @@ import { convertVersions, validFromVersions, validToVersions, -} from "./scripts/convertAtotiToAUIVersions"; +} from "../convertAtotiToAUIVersions"; import { getTreeColumnWidthFromArgs } from "../getTreeColumnWidthFromArgs"; const supportedFileExtension = ["JSON", "IPYNB"]; @@ -151,7 +151,7 @@ yargs if (fileExtension === "JSON") { // Ensure that Atoti versions are not used as versions to migrate content server - migrateContentServer({ + migrateContentServerJson({ inputPath, outputPath, serversPath, diff --git a/src/cli/scripts/migrateContentServer.ts b/src/cli/scripts/migrateContentServerJson.ts similarity index 60% rename from src/cli/scripts/migrateContentServer.ts rename to src/cli/scripts/migrateContentServerJson.ts index 65095506..ec88c0e5 100644 --- a/src/cli/scripts/migrateContentServer.ts +++ b/src/cli/scripts/migrateContentServerJson.ts @@ -1,34 +1,13 @@ import fs from "fs-extra"; import path from "path"; import _capitalize from "lodash/capitalize"; -import _fromPairs from "lodash/fromPairs"; -import _mapValues from "lodash/mapValues"; import { ContentRecord, DataModel } from "@activeviam/activeui-sdk-5.1"; -import { getIndexedDataModel } from "@activeviam/data-model-5.1"; -import { - BehaviorOnError, - OutcomeCounters, - MigrationFunction, -} from "../../migration.types"; -import { getMigrateDashboards } from "../../getMigrateDashboards"; -import { getMigrateSavedWidgets } from "../../getMigrateSavedWidgets"; -import { getMigrateSavedFilters } from "../../getMigrateSavedFilters"; -import { migrate_43_to_50 } from "../../4.3_to_5.0"; -import { migrate_50_to_51 } from "../../5.0_to_5.1"; -import { getContent } from "../../getContent"; +import { BehaviorOnError } from "../../migration.types"; import { AtotiUIFromVersion, AtotiUIToVersion, -} from "./convertAtotiToAUIVersions"; - -const migrationSteps: { - from: string; - to: string; - migrate: MigrationFunction; -}[] = [{ from: "5.0", to: "5.1", migrate: migrate_50_to_51 }]; - -const fromVersions = migrationSteps.map(({ from }) => from); -const toVersions = migrationSteps.map(({ to }) => to); +} from "../../convertAtotiToAUIVersions"; +import { migrateContentServer } from "../../migrateContentServer"; const summaryMessages: { [folderName: string]: { @@ -90,7 +69,10 @@ const summaryMessages: { }, }; -export async function migrateContentServer({ +/** + * Same as {@link migrateContentServer}, but reading its inputs from JSON files. + */ +export async function migrateContentServerJson({ inputPath, outputPath, serversPath, @@ -117,22 +99,6 @@ export async function migrateContentServer({ }): Promise { const contentServer: ContentRecord = await fs.readJSON(inputPath); - const originalDashboardsContent = getContent( - contentServer, - "dashboard", - fromVersion, - ); - const originalWidgetsContent = getContent( - contentServer, - "widget", - fromVersion, - ); - const originalFiltersContent = getContent( - contentServer, - "filter", - fromVersion, - ); - const servers: { [serverKey: string]: { dataModel: DataModel<"raw">; @@ -140,100 +106,17 @@ export async function migrateContentServer({ }; } & { contentServerVersion?: string } = await fs.readJSON(serversPath); - const contentServerVersion = servers.contentServerVersion; - // Later, `servers` is expected to have a structure like `{ [serverKey: string]: { ... } }`. - // Several functions depending on this structure are applied to it: `_mapKeys`, `_findKey`, etc. - // Deleting the "contentServerVersion" attribute from it enables those functions still being applied to `servers` without breaking. - delete servers.contentServerVersion; - - const counters = _fromPairs( - ["dashboards", "widgets", "filters", "folders", "calculated_measures"].map( - (type) => [ - type, - { - success: 0, - partial: 0, - failed: 0, - removed: 0, - }, - ], - ), - // _fromPairs returns a Dictionary. - // In this case, the keys used correspond to the attributes of OutcomeCounters. - ) as OutcomeCounters; - const errorReport = {}; - - // Handle the special case of 4.3 to 5.0 separately, as: - // - the corresponding migration function has a different signature than all others - // - in particular, it is the only migration step with async logic - if (fromVersion === "4.3") { - await migrate_43_to_50(contentServer, { - errorReport, - counters, - servers, - keysOfWidgetPluginsToRemove, - doesReportIncludeStacks, - shouldUpdateFiltersMdx, - treeTableColumnWidth, - }); - } - - const fromVersionIndex = - fromVersion === "4.3" ? 0 : fromVersions.indexOf(fromVersion); - const toVersionIndex = toVersions.indexOf(toVersion); - - const dataModels = _mapValues(servers, ({ dataModel }) => - getIndexedDataModel(dataModel), - ); - - migrationSteps - .slice(fromVersionIndex, toVersionIndex + 1) - .forEach(({ migrate, from, to }) => { - const step = `${from} to ${to}`; - const migrateDashboards = getMigrateDashboards(contentServer, { - originalContent: originalDashboardsContent, - dataModels, - keysOfWidgetPluginsToRemove, - errorReport, - counters, - doesReportIncludeStacks, - behaviorOnError, - step, - }); - - const migrateSavedWidgets = getMigrateSavedWidgets(contentServer, { - originalContent: originalWidgetsContent, - dataModels, - keysOfWidgetPluginsToRemove, - errorReport, - counters, - doesReportIncludeStacks, - behaviorOnError, - step, - }); - - const migrateSavedFilters = getMigrateSavedFilters(contentServer, { - originalContent: originalFiltersContent, - dataModels, - errorReport, - counters, - doesReportIncludeStacks, - behaviorOnError, - step, - }); - - migrate(contentServer, { - migrateDashboards, - migrateSavedWidgets, - migrateSavedFilters, - dataModels, - keysOfWidgetPluginsToRemove, - errorReport, - counters, - doesReportIncludeStacks, - contentServerVersion, - }); - }); + const { counters, errorReport } = await migrateContentServer({ + contentServer, + servers, + fromVersion, + toVersion, + keysOfWidgetPluginsToRemove, + doesReportIncludeStacks, + behaviorOnError, + treeTableColumnWidth, + shouldUpdateFiltersMdx, + }); const { dir } = path.parse(outputPath); diff --git a/src/cli/scripts/migrateNotebook.ts b/src/cli/scripts/migrateNotebook.ts index 7fdc66e2..88120cd6 100644 --- a/src/cli/scripts/migrateNotebook.ts +++ b/src/cli/scripts/migrateNotebook.ts @@ -8,7 +8,7 @@ import { MigrateWidgetCallback } from "../../migration.types"; import { AtotiUIFromVersion, AtotiUIToVersion, -} from "./convertAtotiToAUIVersions"; +} from "../../convertAtotiToAUIVersions"; import { produce } from "immer"; const migrationSteps: { diff --git a/src/cli/scripts/convertAtotiToAUIVersions.test.ts b/src/convertAtotiToAUIVersions.test.ts similarity index 100% rename from src/cli/scripts/convertAtotiToAUIVersions.test.ts rename to src/convertAtotiToAUIVersions.test.ts diff --git a/src/cli/scripts/convertAtotiToAUIVersions.ts b/src/convertAtotiToAUIVersions.ts similarity index 100% rename from src/cli/scripts/convertAtotiToAUIVersions.ts rename to src/convertAtotiToAUIVersions.ts diff --git a/src/migrateContentServer.test.ts b/src/migrateContentServer.test.ts new file mode 100644 index 00000000..3d20ed70 --- /dev/null +++ b/src/migrateContentServer.test.ts @@ -0,0 +1,234 @@ +import { ContentRecord } from "@activeviam/activeui-sdk-5.1"; +import { servers } from "./4.3_to_5.0/__test_resources__/servers"; +import { smallLegacyPivotFolder } from "./4.3_to_5.0/__test_resources__/smallLegacyPivotFolder"; +import { smallLegacyUIFolder } from "./4.3_to_5.0/__test_resources__/smallLegacyUIFolder"; +import { migrateContentServer } from "./migrateContentServer"; +import _cloneDeep from "lodash/cloneDeep"; + +jest.mock(`./4.3_to_5.0/generateId`, () => { + let counter = 0; + return { + generateId: jest.fn(() => { + const id = `00${counter}`; + counter += 1; + + return id; + }), + }; +}); + +describe("migrateContentServer", () => { + it("migrates calculated measures from the /pivot folder to the /ui folder when migrating from 4.3 to 5.0", async () => { + const contentServer: ContentRecord = { + children: { ui: smallLegacyUIFolder, pivot: smallLegacyPivotFolder }, + entry: { + owners: [], + readers: [], + isDirectory: true, + canRead: true, + canWrite: false, + lastEditor: "Freddie Mercury", + timestamp: 0xbeef, + }, + }; + + await migrateContentServer({ + contentServer, + servers, + fromVersion: "4.3", + toVersion: "5.0", + keysOfWidgetPluginsToRemove: [], + doesReportIncludeStacks: false, + shouldUpdateFiltersMdx: true, + behaviorOnError: "keep-original", + }); + + expect(contentServer.children?.ui.children?.calculated_measures) + .toMatchInlineSnapshot(` + { + "children": { + "content": { + "children": { + "000": { + "entry": { + "content": "{"expression":"IIf(IsEmpty(([Booking].[Desk].CurrentMember.Parent, [Measures].[pnl.SUM])), null, [Measures].[pnl.SUM] / ([Booking].[Desk].CurrentMember.Parent, [Measures].[pnl.SUM]))","properties":["FORMAT_STRING=\\"#,###.##%\\""]}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + "001": { + "entry": { + "content": "{"expression":"IIf(IsEmpty(([Currency].[Currency].[ALL].[AllMember], [Measures].[pnl.FOREX])), null, [Measures].[pnl.FOREX] / ([Currency].[Currency].[ALL].[AllMember], [Measures].[pnl.FOREX]))","properties":["FORMAT_STRING=\\"#,###.##%\\""]}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + "002": { + "entry": { + "content": "{"expression":"Count(Descendants([CounterParty].[CounterParty].CurrentMember, [CounterParty].[CounterParty].[CounterPartyGroup]), EXCLUDEEMPTY)","properties":["FORMAT_STRING=\\"#,###.##\\""]}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "ROLE_USER", + ], + "readers": [ + "ROLE_USER", + ], + }, + }, + "structure": { + "children": { + "000": { + "children": { + "000_metadata": { + "entry": { + "content": "{"name":"[Measures].[third calculated measure]"}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + "001": { + "children": { + "001_metadata": { + "entry": { + "content": "{"name":"[Measures].[second calculated measure]"}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + "002": { + "children": { + "002_metadata": { + "entry": { + "content": "{"name":"[Measures].[first calculated measure]"}", + "isDirectory": false, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "admin", + ], + "readers": [ + "admin", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "ROLE_USER", + ], + "readers": [ + "ROLE_USER", + ], + }, + }, + }, + "entry": { + "isDirectory": true, + "owners": [ + "ROLE_CS_ROOT", + ], + "readers": [ + "ROLE_CS_ROOT", + ], + }, + } + `); + }); + it("does not migrate calculated measures when migrating from 4.3 to > 5.0", async () => { + const contentServer: ContentRecord = { + children: { ui: smallLegacyUIFolder, pivot: smallLegacyPivotFolder }, + entry: { + owners: [], + readers: [], + isDirectory: true, + canRead: true, + canWrite: false, + lastEditor: "Freddie Mercury", + timestamp: 0xbeef, + }, + }; + + const contentServerBeforeMigration = _cloneDeep(contentServer); + + await migrateContentServer({ + contentServer, + servers, + fromVersion: "4.3", + toVersion: "5.1", + keysOfWidgetPluginsToRemove: [], + doesReportIncludeStacks: false, + shouldUpdateFiltersMdx: true, + behaviorOnError: "keep-original", + }); + + expect(contentServer.children?.pivot).toStrictEqual( + contentServerBeforeMigration.children?.pivot, + ); + expect(contentServer.children?.ui.children?.calculated_measures).toBe( + undefined, + ); + }); +}); diff --git a/src/migrateContentServer.ts b/src/migrateContentServer.ts new file mode 100644 index 00000000..afb63f53 --- /dev/null +++ b/src/migrateContentServer.ts @@ -0,0 +1,181 @@ +import _fromPairs from "lodash/fromPairs"; +import _mapValues from "lodash/mapValues"; +import { ContentRecord } from "@activeviam/activeui-sdk-5.1"; +import { getIndexedDataModel, DataModel } from "@activeviam/data-model-5.1"; +import { + BehaviorOnError, + OutcomeCounters, + MigrationFunction, +} from "./migration.types"; +import { getMigrateDashboards } from "./getMigrateDashboards"; +import { getMigrateSavedWidgets } from "./getMigrateSavedWidgets"; +import { getMigrateSavedFilters } from "./getMigrateSavedFilters"; +import { migrate_43_to_50 } from "./4.3_to_5.0"; +import { migrate_50_to_51 } from "./5.0_to_5.1"; +import { getContent } from "./getContent"; +import { + AtotiUIFromVersion, + AtotiUIToVersion, +} from "./convertAtotiToAUIVersions"; + +const migrationSteps: { + from: string; + to: string; + migrate: MigrationFunction; +}[] = [{ from: "5.0", to: "5.1", migrate: migrate_50_to_51 }]; + +const fromVersions = migrationSteps.map(({ from }) => from); +const toVersions = migrationSteps.map(({ to }) => to); + +/** + * Migrates `contentServer` from an older version of Atoti UI to a more recent one. + * Also keeps track of the number of migration successes and failures in `counters` and a detailed `errorReport`. + * + * Mutates `contentServer`, `errorReport` and `counters`. + */ +export async function migrateContentServer({ + contentServer, + servers, + fromVersion, + toVersion, + keysOfWidgetPluginsToRemove, + doesReportIncludeStacks, + behaviorOnError, + treeTableColumnWidth, + shouldUpdateFiltersMdx, +}: { + contentServer: ContentRecord; + servers: { + [serverKey: string]: { + dataModel: DataModel<"raw">; + url: string; + }; + } & { contentServerVersion?: string }; + fromVersion: AtotiUIFromVersion; + toVersion: AtotiUIToVersion; + keysOfWidgetPluginsToRemove: string[]; + doesReportIncludeStacks: boolean; + behaviorOnError: BehaviorOnError; + treeTableColumnWidth?: [number, number]; + shouldUpdateFiltersMdx: boolean; +}): Promise<{ + counters: OutcomeCounters; + errorReport: Record; +}> { + const originalDashboardsContent = getContent( + contentServer, + "dashboard", + fromVersion, + ); + const originalWidgetsContent = getContent( + contentServer, + "widget", + fromVersion, + ); + const originalFiltersContent = getContent( + contentServer, + "filter", + fromVersion, + ); + + const contentServerVersion = servers.contentServerVersion; + // Later, `servers` is expected to have a structure like `{ [serverKey: string]: { ... } }`. + // Several functions depending on this structure are applied to it: `_mapKeys`, `_findKey`, etc. + // Deleting the "contentServerVersion" attribute from it enables those functions still being applied to `servers` without breaking. + delete servers.contentServerVersion; + + const counters = _fromPairs( + ["dashboards", "widgets", "filters", "folders", "calculated_measures"].map( + (type) => [ + type, + { + success: 0, + partial: 0, + failed: 0, + removed: 0, + }, + ], + ), + // _fromPairs returns a Dictionary. + // In this case, the keys used correspond to the attributes of OutcomeCounters. + ) as OutcomeCounters; + const errorReport = {}; + + // Handle the special case of 4.3 to 5.0 separately, as: + // - the corresponding migration function has a different signature than all others + // - in particular, it is the only migration step with async logic + if (fromVersion === "4.3") { + await migrate_43_to_50(contentServer, { + errorReport, + counters, + servers, + keysOfWidgetPluginsToRemove, + doesReportIncludeStacks, + shouldUpdateFiltersMdx, + treeTableColumnWidth, + // In Atoti UI 4.3 and 5.1+, saved calculated measures are an Atoti Server concept, and are stored under /pivot in the content server. + // In Atoti UI 5.0, saved calculated measures are an Atoti UI concept, and are stored under /ui in the content server. + // So it's only when the target version is 5.0 that calculated measures need to be migrated. + shouldMigrateCalculatedMeasures: toVersion === "5.0", + }); + } + + const fromVersionIndex = + fromVersion === "4.3" ? 0 : fromVersions.indexOf(fromVersion); + const toVersionIndex = toVersions.indexOf(toVersion); + + const dataModels = _mapValues(servers, ({ dataModel }) => + getIndexedDataModel(dataModel), + ); + + migrationSteps + .slice(fromVersionIndex, toVersionIndex + 1) + .forEach(({ migrate, from, to }) => { + const step = `${from} to ${to}`; + const migrateDashboards = getMigrateDashboards(contentServer, { + originalContent: originalDashboardsContent, + dataModels, + keysOfWidgetPluginsToRemove, + errorReport, + counters, + doesReportIncludeStacks, + behaviorOnError, + step, + }); + + const migrateSavedWidgets = getMigrateSavedWidgets(contentServer, { + originalContent: originalWidgetsContent, + dataModels, + keysOfWidgetPluginsToRemove, + errorReport, + counters, + doesReportIncludeStacks, + behaviorOnError, + step, + }); + + const migrateSavedFilters = getMigrateSavedFilters(contentServer, { + originalContent: originalFiltersContent, + dataModels, + errorReport, + counters, + doesReportIncludeStacks, + behaviorOnError, + step, + }); + + migrate(contentServer, { + migrateDashboards, + migrateSavedWidgets, + migrateSavedFilters, + dataModels, + keysOfWidgetPluginsToRemove, + errorReport, + counters, + doesReportIncludeStacks, + contentServerVersion, + }); + }); + + return { errorReport, counters }; +} From 79c418fd4aa0ad1d06dc05f3efcc7f2a8fb58d0d Mon Sep 17 00:00:00 2001 From: Antoine Tissier Date: Thu, 16 May 2024 13:15:54 +0200 Subject: [PATCH 2/4] prettier --- src/5.0_to_5.1/migrate_5.0_to_5.1.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/5.0_to_5.1/migrate_5.0_to_5.1.ts b/src/5.0_to_5.1/migrate_5.0_to_5.1.ts index 6d945a2d..843c80c7 100644 --- a/src/5.0_to_5.1/migrate_5.0_to_5.1.ts +++ b/src/5.0_to_5.1/migrate_5.0_to_5.1.ts @@ -72,7 +72,7 @@ export const migrate_50_to_51: MigrationFunction = ( doesReportIncludeStacks, step: "5.0 to 5.1", contentServerVersion, - dataModels + dataModels, }); migrateSavedFilters( From b15863762567779f084a88609975c70b3bbe6d23 Mon Sep 17 00:00:00 2001 From: Antoine Tissier Date: Thu, 16 May 2024 13:50:16 +0200 Subject: [PATCH 3/4] space --- src/migrateContentServer.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/migrateContentServer.test.ts b/src/migrateContentServer.test.ts index 3d20ed70..c0667927 100644 --- a/src/migrateContentServer.test.ts +++ b/src/migrateContentServer.test.ts @@ -197,6 +197,7 @@ describe("migrateContentServer", () => { } `); }); + it("does not migrate calculated measures when migrating from 4.3 to > 5.0", async () => { const contentServer: ContentRecord = { children: { ui: smallLegacyUIFolder, pivot: smallLegacyPivotFolder }, From 3f9b52b49896a2a4165af63d46fc0d8ae587500a Mon Sep 17 00:00:00 2001 From: Antoine Tissier Date: Fri, 17 May 2024 14:30:21 +0200 Subject: [PATCH 4/4] further refactoring --- bin.js | 2 +- src/{cli => }/bin.ts | 74 ++++++++++++----- ...entServerJson.ts => logMigrationReport.ts} | 79 ++++++------------- src/{cli/scripts => }/migrateNotebook.ts | 34 +++----- webpack.config.js | 6 +- 5 files changed, 92 insertions(+), 103 deletions(-) rename src/{cli => }/bin.ts (76%) rename src/{cli/scripts/migrateContentServerJson.ts => logMigrationReport.ts} (75%) rename src/{cli/scripts => }/migrateNotebook.ts (75%) diff --git a/bin.js b/bin.js index fe392e20..6e17a457 100755 --- a/bin.js +++ b/bin.js @@ -1,2 +1,2 @@ #!/usr/bin/env node -require("./dist/cli/bin.js"); +require("./dist/bin.js"); diff --git a/src/cli/bin.ts b/src/bin.ts similarity index 76% rename from src/cli/bin.ts rename to src/bin.ts index 6f5d0611..a3d0b2f2 100644 --- a/src/cli/bin.ts +++ b/src/bin.ts @@ -1,15 +1,19 @@ import yargs from "yargs"; -import { BehaviorOnError } from "../migration.types"; -import { migrateContentServerJson } from "./scripts/migrateContentServerJson"; -import { migrateNotebook } from "./scripts/migrateNotebook"; +import fs from "fs-extra"; +import path from "path"; +import { BehaviorOnError } from "./migration.types"; +import { migrateNotebook } from "./migrateNotebook"; import { convertFromVersion, convertToVersion, convertVersions, validFromVersions, validToVersions, -} from "../convertAtotiToAUIVersions"; -import { getTreeColumnWidthFromArgs } from "../getTreeColumnWidthFromArgs"; +} from "./convertAtotiToAUIVersions"; +import { getTreeColumnWidthFromArgs } from "./getTreeColumnWidthFromArgs"; +import { migrateContentServer } from "./migrateContentServer"; +import { DataModel, ContentRecord } from "@activeviam/activeui-sdk-5.1"; +import { logMigrationReport } from "./logMigrationReport"; const supportedFileExtension = ["JSON", "IPYNB"]; @@ -127,7 +131,7 @@ yargs }); args.implies("stack", "debug"); }, - ({ + async ({ inputPath, outputPath, serversPath, @@ -141,6 +145,7 @@ yargs onError, }) => { const doesReportIncludeStacks = stack; + const behaviorOnError = onError; const fileExtension = getFileExtension(inputPath); const { fromVersion: validFromVersion, toVersion: validToVersion } = @@ -149,32 +154,59 @@ yargs toVersion, }); + const fileToMigrate: ContentRecord = await fs.readJSON(inputPath); + + const servers: { + [serverKey: string]: { + dataModel: DataModel<"raw">; + url: string; + }; + } & { contentServerVersion?: string } = await fs.readJSON(serversPath); + if (fileExtension === "JSON") { - // Ensure that Atoti versions are not used as versions to migrate content server - migrateContentServerJson({ - inputPath, - outputPath, - serversPath, + const { counters, errorReport } = await migrateContentServer({ + contentServer: fileToMigrate, + servers, fromVersion: validFromVersion, toVersion: validToVersion, - removeWidgets, - debug, + keysOfWidgetPluginsToRemove: removeWidgets, doesReportIncludeStacks, - onError, + behaviorOnError: onError, treeTableColumnWidth: treeColumnWidth ? getTreeColumnWidthFromArgs(treeColumnWidth) : undefined, shouldUpdateFiltersMdx: updateFiltersMdx === undefined ? true : updateFiltersMdx, }); + + const { dir } = path.parse(outputPath); + await Promise.all([ + fs.writeJSON(outputPath, fileToMigrate, { spaces: 2 }), + logMigrationReport({ + counters, + errorReport, + debug, + doesReportIncludeStacks, + migrationOutputDirectory: dir, + behaviorOnError, + }), + ]); } else { - migrateNotebook({ - inputPath, - outputPath, - serversPath, - fromVersion: validFromVersion, - toVersion: validToVersion, - }); + const { numberOfMigratedWidgets, numberOfFailures } = + await migrateNotebook({ + notebook: fileToMigrate, + servers, + fromVersion: validFromVersion, + toVersion: validToVersion, + }); + + await fs.writeJSON(outputPath, fileToMigrate, { spaces: 2 }); + console.log( + `- Succesfully migrated ${numberOfMigratedWidgets} widget(s).`, + ); + if (numberOfFailures > 0) { + console.log(`- Failed to migrate ${numberOfFailures} widget(s)`); + } } }, ) diff --git a/src/cli/scripts/migrateContentServerJson.ts b/src/logMigrationReport.ts similarity index 75% rename from src/cli/scripts/migrateContentServerJson.ts rename to src/logMigrationReport.ts index ec88c0e5..6b815d0c 100644 --- a/src/cli/scripts/migrateContentServerJson.ts +++ b/src/logMigrationReport.ts @@ -1,13 +1,12 @@ import fs from "fs-extra"; import path from "path"; import _capitalize from "lodash/capitalize"; -import { ContentRecord, DataModel } from "@activeviam/activeui-sdk-5.1"; -import { BehaviorOnError } from "../../migration.types"; + import { - AtotiUIFromVersion, - AtotiUIToVersion, -} from "../../convertAtotiToAUIVersions"; -import { migrateContentServer } from "../../migrateContentServer"; + BehaviorOnError, + ErrorReport, + OutcomeCounters, +} from "./migration.types"; const summaryMessages: { [folderName: string]: { @@ -70,58 +69,24 @@ const summaryMessages: { }; /** - * Same as {@link migrateContentServer}, but reading its inputs from JSON files. + * Logs the outcome of the migration in the console, which includes the number of successful/failed individual resource migrations. + * If `debug` is set to `true`, also logs the detailed `errorReport` in a `report.json` file under `migrationOutputDirectory`. */ -export async function migrateContentServerJson({ - inputPath, - outputPath, - serversPath, - fromVersion, - toVersion, - removeWidgets: keysOfWidgetPluginsToRemove, +export async function logMigrationReport({ + counters, + errorReport, debug, doesReportIncludeStacks, - onError: behaviorOnError, - treeTableColumnWidth, - shouldUpdateFiltersMdx, + behaviorOnError, + migrationOutputDirectory, }: { - inputPath: string; - outputPath: string; - serversPath: string; - fromVersion: AtotiUIFromVersion; - toVersion: AtotiUIToVersion; - removeWidgets: string[]; + counters: OutcomeCounters; + errorReport: ErrorReport; debug: boolean; doesReportIncludeStacks: boolean; - onError: BehaviorOnError; - treeTableColumnWidth?: [number, number]; - shouldUpdateFiltersMdx: boolean; + behaviorOnError: BehaviorOnError; + migrationOutputDirectory: string; }): Promise { - const contentServer: ContentRecord = await fs.readJSON(inputPath); - - const servers: { - [serverKey: string]: { - dataModel: DataModel<"raw">; - url: string; - }; - } & { contentServerVersion?: string } = await fs.readJSON(serversPath); - - const { counters, errorReport } = await migrateContentServer({ - contentServer, - servers, - fromVersion, - toVersion, - keysOfWidgetPluginsToRemove, - doesReportIncludeStacks, - behaviorOnError, - treeTableColumnWidth, - shouldUpdateFiltersMdx, - }); - - const { dir } = path.parse(outputPath); - - await fs.writeJSON(outputPath, contentServer, { spaces: 2 }); - console.log("--------- END OF CONTENT MIGRATION ---------"); Object.entries(counters) @@ -159,7 +124,7 @@ export async function migrateContentServerJson({ ) { if (!debug) { console.log(`For more information about the errors that occurred, rerun the command with the \`--debug\` option. -This will output a file named \`report.json\` containing the error messages.`); + This will output a file named \`report.json\` containing the error messages.`); } else { console.log( `See report.json for more information about the errors that occurred.`, @@ -175,8 +140,12 @@ This will output a file named \`report.json\` containing the error messages.`); console.log("--------------------------------------------"); if (errorReport && debug) { - await fs.writeJSON(path.join(...dir, "report.json"), errorReport, { - spaces: 2, - }); + await fs.writeJSON( + path.join(...migrationOutputDirectory, "report.json"), + errorReport, + { + spaces: 2, + }, + ); } } diff --git a/src/cli/scripts/migrateNotebook.ts b/src/migrateNotebook.ts similarity index 75% rename from src/cli/scripts/migrateNotebook.ts rename to src/migrateNotebook.ts index 88120cd6..2de1d520 100644 --- a/src/cli/scripts/migrateNotebook.ts +++ b/src/migrateNotebook.ts @@ -1,14 +1,13 @@ -import { migrateWidget } from "../../5.0_to_5.1/migrateWidget"; -import fs from "fs-extra"; +import { migrateWidget } from "./5.0_to_5.1/migrateWidget"; import { DataModel, getIndexedDataModel } from "@activeviam/data-model-5.1"; import _mapValues from "lodash/mapValues"; import { serializeWidgetState } from "@activeviam/activeui-sdk-5.1"; import { deserializeWidgetState } from "@activeviam/activeui-sdk-5.0"; -import { MigrateWidgetCallback } from "../../migration.types"; +import { MigrateWidgetCallback } from "./migration.types"; import { AtotiUIFromVersion, AtotiUIToVersion, -} from "../../convertAtotiToAUIVersions"; +} from "./convertAtotiToAUIVersions"; import { produce } from "immer"; const migrationSteps: { @@ -21,24 +20,18 @@ const migrationSteps: { * Migrates the Atoti UI widgets from 5.0 to 5.1 of an Atoti Jupter Notebook. */ export const migrateNotebook = async ({ - inputPath, - outputPath, - serversPath, + notebook, + servers, fromVersion, toVersion, }: { - inputPath: string; - outputPath: string; - serversPath: string; + notebook: any; + servers: { + [serverKey: string]: { dataModel: DataModel<"raw">; url: string }; + }; fromVersion: AtotiUIFromVersion; toVersion: AtotiUIToVersion; -}): Promise => { - const notebook = await fs.readJSON(inputPath); - - const servers: { - [serverKey: string]: { dataModel: DataModel<"raw">; url: string }; - } = await fs.readJSON(serversPath); - +}): Promise<{ numberOfMigratedWidgets: number; numberOfFailures: number }> => { const dataModels = _mapValues(servers, ({ dataModel }) => getIndexedDataModel(dataModel), ); @@ -77,10 +70,5 @@ export const migrateNotebook = async ({ } } - await fs.writeJSON(outputPath, notebook, { spaces: 2 }); - - console.log(`- Succesfully migrated ${numberOfMigratedWidgets} widget(s).`); - if (numberOfFailures > 0) { - console.log(`- Failed to migrate ${numberOfFailures} widget(s)`); - } + return { numberOfMigratedWidgets, numberOfFailures }; }; diff --git a/webpack.config.js b/webpack.config.js index c0373390..443cd44e 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -7,7 +7,7 @@ const webpack = require("webpack"); // This allows the functions exported by this package to be run smoothly in a Node.js environment (see https://support.activeviam.com/jira/browse/UI-6165). module.exports = { entry: { - bin: "./src/cli/bin.ts", + bin: "./src/bin.ts", }, module: { rules: [ @@ -39,8 +39,8 @@ module.exports = { output: { hashFunction: "xxhash64", globalObject: "window", - filename: (pathData) => { - return pathData.chunk.name === "bin" ? "cli/[name].js" : "[name].js"; + filename: () => { + return "[name].js"; }, path: path.resolve(__dirname, "dist"), },