Biigint ftw wave 2 (#9170)

* BigInt for connectors DB * ✨ * tmp * Fix connectors foreign key constraint * Fix safety flag on migration * 👕
dust-tt · Dec 5, 2024 · 3d0a916 · 3d0a916
1 parent 67a9c5a
commit 3d0a916
Show file tree

Hide file tree

Showing 16 changed files with 262 additions and 645 deletions.
diff --git a/connectors/migrations/db/migration_40.sql b/connectors/migrations/db/migration_40.sql
@@ -0,0 +1,62 @@
+\set ON_ERROR_STOP on
+
+-- Check if the safety flag variable exists
+\if :i_know_what_i_do
+-- your migration code here
+\else
+\echo 'Safety flag not set. Run with: psql -v i_know_what_i_do=true -f migration_40.sql'
+\q
+\endif
+
+DO $$
+DECLARE
+    r RECORD;
+BEGIN
+    -- First convert all PKs
+    FOR r IN (
+        SELECT
+            t.table_name,
+            c.column_name
+        FROM information_schema.tables t
+        JOIN information_schema.columns c
+            ON t.table_name = c.table_name
+            AND t.table_schema = c.table_schema
+        JOIN information_schema.key_column_usage k
+            ON c.column_name = k.column_name
+            AND c.table_name = k.table_name
+        WHERE c.data_type = 'integer'
+            AND t.table_schema = 'public'
+            AND EXISTS (
+                SELECT 1
+                FROM information_schema.table_constraints tc
+                WHERE tc.constraint_type = 'PRIMARY KEY'
+                    AND tc.table_name = t.table_name
+                    AND tc.constraint_name = k.constraint_name
+            )
+    ) LOOP
+        RAISE NOTICE 'Converting PK column % in table % to bigint', r.column_name, r.table_name;
+        EXECUTE format('ALTER TABLE %I ALTER COLUMN %I TYPE bigint', r.table_name, r.column_name);
+    END LOOP;
+
+    -- Then convert all FKs
+    FOR r IN (
+        SELECT DISTINCT
+            t.table_name,
+            c.column_name
+        FROM information_schema.tables t
+        JOIN information_schema.columns c
+            ON t.table_name = c.table_name
+            AND t.table_schema = c.table_schema
+        JOIN information_schema.key_column_usage k
+            ON c.column_name = k.column_name
+            AND c.table_name = k.table_name
+        JOIN information_schema.table_constraints tc
+            ON tc.constraint_name = k.constraint_name
+        WHERE c.data_type = 'integer'
+            AND t.table_schema = 'public'
+            AND tc.constraint_type = 'FOREIGN KEY'
+    ) LOOP
+        RAISE NOTICE 'Converting FK column % in table % to bigint', r.column_name, r.table_name;
+        EXECUTE format('ALTER TABLE %I ALTER COLUMN %I TYPE bigint', r.table_name, r.column_name);
+    END LOOP;
+END $$;
diff --git a/connectors/src/lib/models/confluence.ts b/connectors/src/lib/models/confluence.ts
@@ -1,19 +1,11 @@
-import type {
-  CreationOptional,
-  ForeignKey,
-  InferAttributes,
-  InferCreationAttributes,
-} from "sequelize";
-import { DataTypes, Model } from "sequelize";
+import type { CreationOptional, ForeignKey } from "sequelize";
+import { DataTypes } from "sequelize";
 
 import { sequelizeConnection } from "@connectors/resources/storage";
 import { ConnectorModel } from "@connectors/resources/storage/models/connector_model";
+import { BaseModel } from "@connectors/resources/storage/wrappers";
 
-export class ConfluenceConfiguration extends Model<
-  InferAttributes<ConfluenceConfiguration>,
-  InferCreationAttributes<ConfluenceConfiguration>
-> {
-  declare id: CreationOptional<number>;
+export class ConfluenceConfiguration extends BaseModel<ConfluenceConfiguration> {
   declare createdAt: CreationOptional<Date>;
   declare updatedAt: CreationOptional<Date>;
 
@@ -25,11 +17,6 @@ export class ConfluenceConfiguration extends Model<
 }
 ConfluenceConfiguration.init(
   {
-    id: {
-      type: DataTypes.INTEGER,
-      autoIncrement: true,
-      primaryKey: true,
-    },
     cloudId: {
       type: DataTypes.STRING,
       allowNull: false,
@@ -65,11 +52,7 @@ ConfluenceConfiguration.init(
 ConnectorModel.hasOne(ConfluenceConfiguration);
 
 // ConfluenceSpace stores the global spaces selected by the user to sync.
-export class ConfluenceSpace extends Model<
-  InferAttributes<ConfluenceSpace>,
-  InferCreationAttributes<ConfluenceSpace>
-> {
-  declare id: CreationOptional<number>;
+export class ConfluenceSpace extends BaseModel<ConfluenceSpace> {
   declare createdAt: CreationOptional<Date>;
   declare updatedAt: CreationOptional<Date>;
   declare connectorId: ForeignKey<ConnectorModel["id"]>;
@@ -79,15 +62,6 @@ export class ConfluenceSpace extends Model<
 }
 ConfluenceSpace.init(
   {
-    id: {
-      type: DataTypes.INTEGER,
-      autoIncrement: true,
-      primaryKey: true,
-    },
-    connectorId: {
-      type: DataTypes.INTEGER,
-      allowNull: false,
-    },
     createdAt: {
       type: DataTypes.DATE,
       allowNull: false,
@@ -117,14 +91,13 @@ ConfluenceSpace.init(
     indexes: [{ fields: ["connectorId", "spaceId"], unique: true }],
   }
 );
-ConnectorModel.hasOne(ConfluenceSpace);
+ConnectorModel.hasOne(ConfluenceSpace, {
+  foreignKey: "connectorId",
+  onDelete: "RESTRICT",
+});
 
 // ConfluencePages stores the pages.
-export class ConfluencePage extends Model<
-  InferAttributes<ConfluencePage>,
-  InferCreationAttributes<ConfluencePage>
-> {
-  declare id: CreationOptional<number>;
+export class ConfluencePage extends BaseModel<ConfluencePage> {
   declare createdAt: CreationOptional<Date>;
   declare updatedAt: CreationOptional<Date>;
   declare lastVisitedAt: CreationOptional<Date>;
@@ -141,11 +114,6 @@ export class ConfluencePage extends Model<
 }
 ConfluencePage.init(
   {
-    id: {
-      type: DataTypes.INTEGER,
-      autoIncrement: true,
-      primaryKey: true,
-    },
     createdAt: {
       type: DataTypes.DATE,
       allowNull: false,

diff --git a/connectors/src/lib/models/github.ts b/connectors/src/lib/models/github.ts
@@ -1,19 +1,11 @@
-import type {
-  CreationOptional,
-  ForeignKey,
-  InferAttributes,
-  InferCreationAttributes,
-} from "sequelize";
-import { DataTypes, Model } from "sequelize";
+import type { CreationOptional, ForeignKey } from "sequelize";
+import { DataTypes } from "sequelize";
 
 import { sequelizeConnection } from "@connectors/resources/storage";
 import { ConnectorModel } from "@connectors/resources/storage/models/connector_model";
+import { BaseModel } from "@connectors/resources/storage/wrappers";
 
-export class GithubConnectorState extends Model<
-  InferAttributes<GithubConnectorState>,
-  InferCreationAttributes<GithubConnectorState>
-> {
-  declare id: CreationOptional<number>;
+export class GithubConnectorState extends BaseModel<GithubConnectorState> {
   declare createdAt: CreationOptional<Date>;
   declare updatedAt: CreationOptional<Date>;
 
@@ -25,11 +17,6 @@ export class GithubConnectorState extends Model<
 }
 GithubConnectorState.init(
   {
-    id: {
-      type: DataTypes.INTEGER,
-      autoIncrement: true,
-      primaryKey: true,
-    },
     createdAt: {
       type: DataTypes.DATE,
       allowNull: false,
@@ -65,11 +52,7 @@ GithubConnectorState.init(
 );
 ConnectorModel.hasOne(GithubConnectorState);
 
-export class GithubIssue extends Model<
-  InferAttributes<GithubIssue>,
-  InferCreationAttributes<GithubIssue>
-> {
-  declare id: CreationOptional<number>;
+export class GithubIssue extends BaseModel<GithubIssue> {
   declare createdAt: CreationOptional<Date>;
   declare updatedAt: CreationOptional<Date>;
 
@@ -82,11 +65,6 @@ export class GithubIssue extends Model<
 }
 GithubIssue.init(
   {
-    id: {
-      type: DataTypes.INTEGER,
-      autoIncrement: true,
-      primaryKey: true,
-    },
     createdAt: {
       type: DataTypes.DATE,
       allowNull: false,
@@ -122,11 +100,7 @@ GithubIssue.init(
 );
 ConnectorModel.hasMany(GithubIssue);
 
-export class GithubDiscussion extends Model<
-  InferAttributes<GithubDiscussion>,
-  InferCreationAttributes<GithubDiscussion>
-> {
-  declare id: CreationOptional<number>;
+export class GithubDiscussion extends BaseModel<GithubDiscussion> {
   declare createdAt: CreationOptional<Date>;
   declare updatedAt: CreationOptional<Date>;
 
@@ -137,11 +111,6 @@ export class GithubDiscussion extends Model<
 }
 GithubDiscussion.init(
   {
-    id: {
-      type: DataTypes.INTEGER,
-      autoIncrement: true,
-      primaryKey: true,
-    },
     createdAt: {
       type: DataTypes.DATE,
       allowNull: false,
@@ -173,11 +142,7 @@ GithubDiscussion.init(
 );
 ConnectorModel.hasMany(GithubDiscussion);
 
-export class GithubCodeRepository extends Model<
-  InferAttributes<GithubCodeRepository>,
-  InferCreationAttributes<GithubCodeRepository>
-> {
-  declare id: CreationOptional<number>;
+export class GithubCodeRepository extends BaseModel<GithubCodeRepository> {
   declare createdAt: CreationOptional<Date>;
   declare updatedAt: CreationOptional<Date>;
   declare lastSeenAt: CreationOptional<Date>;
@@ -194,11 +159,6 @@ export class GithubCodeRepository extends Model<
 }
 GithubCodeRepository.init(
   {
-    id: {
-      type: DataTypes.INTEGER,
-      autoIncrement: true,
-      primaryKey: true,
-    },
     createdAt: {
       type: DataTypes.DATE,
       allowNull: false,
@@ -249,11 +209,7 @@ GithubCodeRepository.init(
 );
 ConnectorModel.hasMany(GithubCodeRepository);
 
-export class GithubCodeFile extends Model<
-  InferAttributes<GithubCodeFile>,
-  InferCreationAttributes<GithubCodeFile>
-> {
-  declare id: CreationOptional<number>;
+export class GithubCodeFile extends BaseModel<GithubCodeFile> {
   declare createdAt: CreationOptional<Date>;
   declare updatedAt: CreationOptional<Date>;
   declare lastSeenAt: CreationOptional<Date>;
@@ -271,11 +227,6 @@ export class GithubCodeFile extends Model<
 }
 GithubCodeFile.init(
   {
-    id: {
-      type: DataTypes.INTEGER,
-      autoIncrement: true,
-      primaryKey: true,
-    },
     createdAt: {
       type: DataTypes.DATE,
       allowNull: false,
@@ -332,11 +283,7 @@ GithubCodeFile.init(
 );
 ConnectorModel.hasMany(GithubCodeFile);
 
-export class GithubCodeDirectory extends Model<
-  InferAttributes<GithubCodeDirectory>,
-  InferCreationAttributes<GithubCodeDirectory>
-> {
-  declare id: CreationOptional<number>;
+export class GithubCodeDirectory extends BaseModel<GithubCodeDirectory> {
   declare createdAt: CreationOptional<Date>;
   declare updatedAt: CreationOptional<Date>;
   declare lastSeenAt: CreationOptional<Date>;
@@ -353,11 +300,6 @@ export class GithubCodeDirectory extends Model<
 }
 GithubCodeDirectory.init(
   {
-    id: {
-      type: DataTypes.INTEGER,
-      autoIncrement: true,
-      primaryKey: true,
-    },
     createdAt: {
       type: DataTypes.DATE,
       allowNull: false,