UI-9435 - Better handling of calculated measures (#126)

* UI-9435 - Better handling of calculated measures

* prettier

* space

* further refactoring

bin.js

@@ -1,2 +1,2 @@
 #!/usr/bin/env node
-require("./dist/cli/bin.js");
+require("./dist/bin.js");

src/4.3_to_5.0/migrate_43_to_50.test.ts

@@ -73,6 +73,7 @@ describe("migrate_43_to_50", () => {
       servers,
       doesReportIncludeStacks: false,
       shouldUpdateFiltersMdx: true,
+      shouldMigrateCalculatedMeasures: true,
     });
     const migratedUIFolder = contentServer.children?.ui;
     expect(migratedUIFolder).toMatchSnapshot();
@@ -88,6 +89,7 @@ describe("migrate_43_to_50", () => {
       servers,
       doesReportIncludeStacks: false,
       shouldUpdateFiltersMdx: true,
+      shouldMigrateCalculatedMeasures: true,
     });
     const migratedUIFolder = contentServer.children?.ui;
     expect(migratedUIFolder).toMatchSnapshot();
@@ -106,6 +108,7 @@ describe("migrate_43_to_50", () => {
       servers,
       doesReportIncludeStacks: false,
       shouldUpdateFiltersMdx: true,
+      shouldMigrateCalculatedMeasures: true,
     });
 
     const migratedUIFolder = contentServer.children?.ui;
@@ -125,6 +128,7 @@ describe("migrate_43_to_50", () => {
       keysOfWidgetPluginsToRemove,
       doesReportIncludeStacks: false,
       shouldUpdateFiltersMdx: true,
+      shouldMigrateCalculatedMeasures: true,
     });
 
     // In the ActiveUI 4 folder, the file with id `0xb` represents a saved Page Filters widget.
@@ -191,6 +195,7 @@ describe("migrate_43_to_50", () => {
       servers,
       doesReportIncludeStacks: false,
       shouldUpdateFiltersMdx: true,
+      shouldMigrateCalculatedMeasures: true,
     });
 
     const migratedUIFolder = contentServer.children?.ui;
@@ -257,6 +262,7 @@ describe("migrate_43_to_50", () => {
       servers,
       doesReportIncludeStacks: false,
       shouldUpdateFiltersMdx: true,
+      shouldMigrateCalculatedMeasures: true,
     });
 
     const migratedUIFolder = contentServer.children?.ui;

src/4.3_to_5.0/migrate_43_to_50.ts

@@ -216,6 +216,7 @@ export async function migrate_43_to_50(
     doesReportIncludeStacks,
     shouldUpdateFiltersMdx,
     treeTableColumnWidth,
+    shouldMigrateCalculatedMeasures,
   }: {
     errorReport: ErrorReport;
     counters: OutcomeCounters;
@@ -224,6 +225,7 @@ export async function migrate_43_to_50(
     doesReportIncludeStacks: boolean;
     shouldUpdateFiltersMdx: boolean;
     treeTableColumnWidth?: [number, number];
+    shouldMigrateCalculatedMeasures: boolean;
   },
 ): Promise<void> {
   if (contentServer.children?.ui === undefined) {
@@ -492,7 +494,7 @@ export async function migrate_43_to_50(
 
   migratedUIFolder.children = {
     ...migratedUIFolder.children,
-    ...(legacyPivotFolder
+    ...(legacyPivotFolder && shouldMigrateCalculatedMeasures
       ? {
           calculated_measures: await migrateCalculatedMeasures(
             legacyPivotFolder,

src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.test.ts

@@ -4,6 +4,9 @@ import { uiCalculatedMeasuresFolder } from "../__test_resources__/uiCalculatedMe
 import { ErrorReport, OutcomeCounters } from "../../migration.types";
 import _cloneDeep from "lodash/cloneDeep";
 import _fromPairs from "lodash/fromPairs";
+import { sandboxDataModel } from "@activeviam/data-model-5.1/dist/__test_resources__";
+
+const dataModels = { sandbox: sandboxDataModel };
 
 const contentServerForTests = _cloneDeep(contentServer);
 const errorReport: ErrorReport = {};
@@ -45,15 +48,15 @@ migrateSavedCalculatedMeasures({
   counters,
   doesReportIncludeStacks: false,
   step: "5.0 to 5.1",
+  dataModels,
 });
 
 describe("migrateSavedCalculatedMeasures", () => {
-  it("migrates the serialized definitions of all calculated measures created with ActiveUI 5.0 and used in a saved dashboard or saved widget, into ones that are natively supported by ActivePivot", () => {
+  it("migrates the serialized definitions of all calculated measures created with ActiveUI 5.0, into ones that are natively supported by ActivePivot", () => {
     // `uiCalculatedMeasuresFolder` contains 5 calculated measures.
-    // "Exp gamma sum" is not used in any saved widgets or dashboards, it is not migrated.
-    expect(counters.calculated_measures.success).toEqual(4);
-    expect(counters.calculated_measures.failed).toEqual(1);
+    expect(counters.calculated_measures.success).toEqual(5);
 
+    // "Exp gamma sum" is not used in any saved widget or dashboard. By default, it is associated with the first cube in the data model: `EquityDerivativesCube`.
     // "CM in 2 cubes" is used in both `EquityDerivativesCube` and `EquityDerivativesCubeDist`.
     // All others are only used in `EquityDerivativesCube`.
     // "Log pv.SUM" is inside a folder.
@@ -67,6 +70,7 @@ describe("migrateSavedCalculatedMeasures", () => {
       "[Measures].[testo]",
       "[Measures].[new measure*]",
       "[Measures].[Distinct count city]",
+      "[Measures].[Exp gamma sum]",
       "[Measures].[Log pv.SUM]",
       "[Measures].[CM in 2 cubes]",
       "[Measures].[Test calculated measure]",
@@ -127,21 +131,4 @@ describe("migrateSavedCalculatedMeasures", () => {
       contentServerForTests.children!.ui.children!.calculated_measures,
     ).toBeUndefined();
   });
-
-  it("adds a warning to the `errorReport` that a calculated measure has not been migrated if it is not used in any saved widgets or dashboards", () => {
-    expect(errorReport).toStrictEqual({
-      calculated_measures: {
-        "501": {
-          error: {
-            message:
-              'Warning: Calculated measure "Exp gamma sum" was not migrated because it is not currently used in any saved widgets or dashboards.',
-          },
-          folderId: ["a14"],
-          folderName: ["New folder"],
-          name: "Exp gamma sum",
-          step: "5.0 to 5.1",
-        },
-      },
-    });
-  });
 });

src/5.0_to_5.1/calculated-measures/migrateSavedCalculatedMeasures.ts

@@ -7,6 +7,7 @@ import { ErrorReport, OutcomeCounters } from "../../migration.types";
 import { _getFilesAncestry } from "../../_getFilesAncestry";
 import { _serializeError } from "../../_serializeError";
 import { _getMetaData } from "../../_getMetaData";
+import { DataModel } from "@activeviam/activeui-sdk-5.1";
 
 const contentServerWithEmptyPivotCalculatedMeasuresFolder = {
   entry: {
@@ -72,6 +73,7 @@ export function migrateSavedCalculatedMeasures({
   doesReportIncludeStacks,
   step,
   contentServerVersion,
+  dataModels,
 }: {
   contentServer: ContentRecord;
   measureToCubeMapping: { [measureName: string]: string[] };
@@ -80,6 +82,9 @@ export function migrateSavedCalculatedMeasures({
   doesReportIncludeStacks: boolean;
   step: string;
   contentServerVersion?: string;
+  dataModels: {
+    [serverKey: string]: DataModel;
+  };
 }): void {
   const legacyCalculatedMeasuresFolder =
     contentServer.children?.ui?.children?.calculated_measures;
@@ -89,6 +94,10 @@ export function migrateSavedCalculatedMeasures({
     return;
   }
 
+  const catalogs = Object.values(dataModels)[0].catalogs;
+  const cubes = Object.values(catalogs)[0].cubes;
+  const nameOfFirstCube = Object.keys(cubes)[0];
+
   // Make sure that the folder /pivot/entitlements/cm folder exists.
   _defaultsDeep(
     contentServer,
@@ -104,26 +113,12 @@ export function migrateSavedCalculatedMeasures({
     const measureName = _getMetaData(structure, folderId, id).name!;
 
     try {
-      const cubeNames = measureToCubeMapping[measureName];
-      if (!cubeNames) {
-        // The calculated measure is not used in any saved widgets or dashboards.
-        // Do not migrate it.
-        counters.calculated_measures.failed++;
-        _addErrorToReport(errorReport, {
-          contentType: "calculated_measures",
-          folderId,
-          folderName,
-          fileErrorReport: {
-            error: {
-              message: `Warning: Calculated measure "${measureName}" was not migrated because it is not currently used in any saved widgets or dashboards.`,
-            },
-          },
-          fileId: id,
-          name: measureName,
-          step,
-        });
-        return;
-      }
+      const cubeNames = measureToCubeMapping[
+        measureName
+      ] ?? /** The saved calculated measure is not used in any widget.
+       * So it's impossible to infer the cube it is associated with.
+       * Default to the first cube in the data models.
+       */ [nameOfFirstCube];
 
       const migratedContent = migrateSavedCalculatedMeasureContent(
         JSON.parse(record.entry.content),

src/5.0_to_5.1/migrate_5.0_to_5.1.ts

@@ -72,6 +72,7 @@ export const migrate_50_to_51: MigrationFunction = (
     doesReportIncludeStacks,
     step: "5.0 to 5.1",
     contentServerVersion,
+    dataModels,
   });
 
   migrateSavedFilters(

src/cli/bin.ts → src/bin.ts

@@ -1,15 +1,19 @@
 import yargs from "yargs";
-import { BehaviorOnError } from "../migration.types";
-import { migrateContentServer } from "./scripts/migrateContentServer";
-import { migrateNotebook } from "./scripts/migrateNotebook";
+import fs from "fs-extra";
+import path from "path";
+import { BehaviorOnError } from "./migration.types";
+import { migrateNotebook } from "./migrateNotebook";
 import {
   convertFromVersion,
   convertToVersion,
   convertVersions,
   validFromVersions,
   validToVersions,
-} from "./scripts/convertAtotiToAUIVersions";
-import { getTreeColumnWidthFromArgs } from "../getTreeColumnWidthFromArgs";
+} from "./convertAtotiToAUIVersions";
+import { getTreeColumnWidthFromArgs } from "./getTreeColumnWidthFromArgs";
+import { migrateContentServer } from "./migrateContentServer";
+import { DataModel, ContentRecord } from "@activeviam/activeui-sdk-5.1";
+import { logMigrationReport } from "./logMigrationReport";
 
 const supportedFileExtension = ["JSON", "IPYNB"];
 
@@ -127,7 +131,7 @@ yargs
       });
       args.implies("stack", "debug");
     },
-    ({
+    async ({
       inputPath,
       outputPath,
       serversPath,
@@ -141,6 +145,7 @@ yargs
       onError,
     }) => {
       const doesReportIncludeStacks = stack;
+      const behaviorOnError = onError;
 
       const fileExtension = getFileExtension(inputPath);
       const { fromVersion: validFromVersion, toVersion: validToVersion } =
@@ -149,32 +154,59 @@ yargs
           toVersion,
         });
 
+      const fileToMigrate: ContentRecord = await fs.readJSON(inputPath);
+
+      const servers: {
+        [serverKey: string]: {
+          dataModel: DataModel<"raw">;
+          url: string;
+        };
+      } & { contentServerVersion?: string } = await fs.readJSON(serversPath);
+
       if (fileExtension === "JSON") {
-        // Ensure that Atoti versions are not used as versions to migrate content server
-        migrateContentServer({
-          inputPath,
-          outputPath,
-          serversPath,
+        const { counters, errorReport } = await migrateContentServer({
+          contentServer: fileToMigrate,
+          servers,
           fromVersion: validFromVersion,
           toVersion: validToVersion,
-          removeWidgets,
-          debug,
+          keysOfWidgetPluginsToRemove: removeWidgets,
           doesReportIncludeStacks,
-          onError,
+          behaviorOnError: onError,
           treeTableColumnWidth: treeColumnWidth
             ? getTreeColumnWidthFromArgs(treeColumnWidth)
             : undefined,
           shouldUpdateFiltersMdx:
             updateFiltersMdx === undefined ? true : updateFiltersMdx,
         });
+
+        const { dir } = path.parse(outputPath);
+        await Promise.all([
+          fs.writeJSON(outputPath, fileToMigrate, { spaces: 2 }),
+          logMigrationReport({
+            counters,
+            errorReport,
+            debug,
+            doesReportIncludeStacks,
+            migrationOutputDirectory: dir,
+            behaviorOnError,
+          }),
+        ]);
       } else {
-        migrateNotebook({
-          inputPath,
-          outputPath,
-          serversPath,
-          fromVersion: validFromVersion,
-          toVersion: validToVersion,
-        });
+        const { numberOfMigratedWidgets, numberOfFailures } =
+          await migrateNotebook({
+            notebook: fileToMigrate,
+            servers,
+            fromVersion: validFromVersion,
+            toVersion: validToVersion,
+          });
+
+        await fs.writeJSON(outputPath, fileToMigrate, { spaces: 2 });
+        console.log(
+          `- Succesfully migrated ${numberOfMigratedWidgets} widget(s).`,
+        );
+        if (numberOfFailures > 0) {
+          console.log(`- Failed to migrate ${numberOfFailures} widget(s)`);
+        }
       }
     },
   )