🌟 [WIP] DB Migration Tool

tdrive/backend/node/src/cli/cmds/db.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Manage db migrations",
+  command: "db <command>",
+  builder: yargs =>
+    yargs.commandDir("db_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/db_cmds/migrate-db.ts

@@ -0,0 +1,179 @@
+import yargs from "yargs";
+import runWithPlatform from "../../lib/run-with-platform";
+import runWithLoggerLevel from "../../utils/run-with-logger-level";
+import gr from "../../../services/global-resolver";
+type CLIArgs = {
+  name: string;
+};
+
+// eslint-disable-next-line @typescript-eslint/ban-types
+const command: yargs.CommandModule<unknown, CLIArgs> = {
+  command: "migrate",
+  builder: {
+    name: {
+      default: "",
+      type: "string",
+      description: "Entity name to migrate",
+    },
+    output: {
+      default: "",
+      type: "string",
+      description: "Migration output log",
+    },
+  },
+  describe:
+    "command to export everything inside a company (publicly data only available to a new member)",
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  handler: async argv => {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    await runWithPlatform("Re-index", async ({ spinner, platform }) => {
+      return await runWithLoggerLevel(
+        argv.verboseDuringRun
+          ? (argv.verboseDuringRun as number) > 1
+            ? "debug"
+            : "info"
+          : undefined,
+        async () => {
+          const targetEntity = argv.name;
+          const targetService = await gr.services["users"];
+          const entityClass = targetService.repository.entityType;
+          const entity = entityClass.prototype;
+          // console.log("Migrating entity: \n", JSON.stringify(entity), "\n");
+          const schema = JSON.parse(await gr.database.getConnector().migrate(targetEntity));
+          // Define a mapping between JavaScript data types and PostgreSQL data types
+          const dataTypeMapping = {
+            string: "text",
+            number: "bigint",
+            boolean: "boolean",
+            object: "jsonb", // Assuming JSON objects are stored as jsonb in PostgreSQL
+            date: "timestamp", // Assuming dates are stored as timestamps in PostgreSQL
+            array: "jsonb[]", // Assuming arrays are stored as JSON arrays in PostgreSQL
+            float: "real", // Assuming floating-point numbers are stored as real in PostgreSQL
+            double: "double precision", // Assuming double-precision floating-point numbers are stored as double precision in PostgreSQL
+            int: "integer", // Assuming integer numbers are stored as integer in PostgreSQL
+            bigint: "bigint", // Assuming big integer numbers are stored as bigint in PostgreSQL
+            smallint: "smallint", // Assuming small integer numbers are stored as smallint in PostgreSQL
+            decimal: "numeric", // Assuming decimal numbers are stored as numeric in PostgreSQL
+            encoded_json: "text", // Assuming encoded JSON objects are stored as text in PostgreSQL
+          };
+          console.log("\n ##################################### \n");
+
+          // Compare schema with entity
+          const addedColumns = [];
+          const deletedColumns = [];
+          const changedColumns = [];
+
+          // Check for added columns and renamed columns
+          Object.keys(entity._columns).forEach(columnName => {
+            const columnExistsInSchema = schema.some(
+              schemaField => schemaField.column_name === columnName,
+            );
+            if (!columnExistsInSchema) {
+              addedColumns.push(columnName);
+            } else {
+              const schemaField = schema.find(field => field.column_name === columnName);
+              const entityField = entity._columns[columnName];
+
+              // Check if the column has a 'rename' option
+              const renameOption = entityField.options && entityField.options.rename;
+              const renamedColumnName = renameOption ? renameOption : columnName;
+              const columnAlreadyRenamed = schema.some(
+                schemaField => schemaField.column_name === renamedColumnName,
+              );
+              if (columnAlreadyRenamed && renameOption) {
+                console.log(
+                  `⚠️ Column '${columnName}' renamed to '${renamedColumnName}, please update the entity and remove the 'rename' option from the column definition: ${columnAlreadyRenamed}, ${renameOption}`,
+                );
+              }
+
+              // Map JavaScript data types to PostgreSQL data types if needed
+              const mappedEntityType = dataTypeMapping[entityField.type] || entityField.type;
+
+              if (
+                (schemaField.column_name !== renamedColumnName && !columnAlreadyRenamed) ||
+                schemaField.data_type !== mappedEntityType
+              ) {
+                changedColumns.push({
+                  columnName,
+                  renamedColumnName,
+                  schemaType: schemaField.data_type,
+                  entityType: mappedEntityType,
+                });
+              }
+            }
+          });
+
+          // Check for deleted columns (including renamed columns)
+          schema.forEach(schemaField => {
+            const columnName = schemaField.column_name;
+            const columnExistsInEntity = entity._columns.hasOwnProperty(columnName);
+            if (!columnExistsInEntity) {
+              // Check if the column was renamed in the entity options
+              const renamedColumn = Object.values(entity._columns).find(
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                (field: any) => field?.options && field.options.rename === columnName,
+              );
+              if (!renamedColumn) {
+                deletedColumns.push(columnName);
+              }
+            }
+          });
+
+          // Output changes
+          console.log("\nChanges detected:\n");
+
+          if (addedColumns.length > 0) {
+            console.log("Added columns:");
+            addedColumns.forEach(columnName => console.log(`- ${columnName}`));
+          } else {
+            console.log("No new columns added");
+          }
+
+          if (deletedColumns.length > 0) {
+            console.log("\nDeleted columns:");
+            deletedColumns.forEach(columnName => console.log(`- ${columnName}`));
+          } else {
+            console.log("No columns deleted");
+          }
+
+          if (changedColumns.length > 0) {
+            console.log("\nChanged columns:");
+            changedColumns.forEach(({ columnName, renamedColumnName, schemaType, entityType }) => {
+              if (columnName !== renamedColumnName) {
+                console.log(`- Column '${columnName}' renamed to '${renamedColumnName}'`);
+              }
+              if (schemaType !== entityType) {
+                console.log(
+                  `- Column '${columnName}' type changed from '${schemaType}' to '${entityType}'`,
+                );
+              }
+            });
+          } else {
+            console.log("No columns changed");
+          }
+
+          // Show appropriate PostgreSQL queries
+          console.log("\nPostgreSQL Queries:");
+          addedColumns.forEach(columnName =>
+            console.log(
+              `ALTER TABLE public."${targetEntity}" ADD COLUMN ${columnName} <data_type>;`,
+            ),
+          );
+          deletedColumns.forEach(columnName =>
+            console.log(`ALTER TABLE public."${targetEntity}" DROP COLUMN ${columnName};`),
+          );
+          changedColumns.forEach(({ columnName, renamedColumnName }) => {
+            if (columnName !== renamedColumnName) {
+              console.log(
+                `ALTER TABLE public."${targetEntity}" RENAME COLUMN ${columnName} TO ${renamedColumnName};`,
+              );
+            }
+          });
+          console.log("\n");
+        },
+      );
+    });
+  },
+};
+
+export default command;

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts

@@ -13,6 +13,8 @@ export abstract class AbstractConnector<T extends ConnectionOptions> implements
 
   abstract drop(): Promise<this>;
 
+  abstract migrate(name: string): Promise<string>;
+
   abstract createTable(
     entity: EntityDefinition,
     columns: { [name: string]: ColumnDefinition },

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts

@@ -74,6 +74,10 @@ export class CassandraConnector extends AbstractConnector<CassandraConnectionOpt
     return this;
   }
 
+  async migrate(name: string): Promise<string> {
+    return `checking ${name}: nothing changed...`;
+  }
+
   createKeyspace(): Promise<cassandra.types.ResultSet> {
     const query = `CREATE KEYSPACE IF NOT EXISTS ${this.options.keyspace} WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1': '2'} AND durable_writes = true;`;
     logger.info(query);

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts

@@ -27,6 +27,12 @@ export interface Connector extends Initializable {
    */
   disconnect(): Promise<this>;
 
+  /**
+   * Migrate the database
+   */
+
+  migrate(name: string): Promise<string>;
+
   /**
    * Get the type of connector
    */

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts

@@ -25,6 +25,10 @@ export class MongoConnector extends AbstractConnector<MongoConnectionOptions> {
     return this;
   }
 
+  async migrate(name: string): Promise<string> {
+    return `checking ${name}: nothing changed...`;
+  }
+
   async connect(): Promise<this> {
     if (this.client) {
       return this;

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts

@@ -56,6 +56,15 @@ export class PostgresConnector extends AbstractConnector<PostgresConnectionOptio
     return this;
   }
 
+  async migrate(name: string): Promise<string> {
+    const query = `SELECT column_name, data_type, character_maximum_length, is_nullable
+                    FROM information_schema.columns
+                    WHERE table_name = '${name}';
+    `;
+    const tableSchema = await this.client.query(query);
+    return `${JSON.stringify(tableSchema.rows)}`;
+  }
+
   async init(): Promise<this> {
     if (!this.client) {
       await this.connect();

tdrive/backend/node/src/core/platform/services/database/services/orm/types.ts

@@ -25,6 +25,7 @@ export type ColumnOptions = {
   order?: "ASC" | "DESC";
   generator?: ColumnType;
   onUpsert?: (value: any) => any;
+  rename?: string;
 };
 
 export type ColumnType =