feat(resource-full-text-search): add text vector columns and GIN indexes

connectors/migrations/20230828_notion_pages_title_search_vector.ts

@@ -0,0 +1,22 @@
+import { sequelize_conn } from "@connectors/lib/models";
+
+async function main() {
+  await sequelize_conn.query(`
+    UPDATE "notion_pages"
+    SET "titleSearchVector" = to_tsvector('english', coalesce("title", ''));
+  `);
+  await sequelize_conn.query(`
+    UPDATE "notion_databases"
+    SET "titleSearchVector" = to_tsvector('english', coalesce("title", ''));
+  `);
+}
+
+main()
+  .then(() => {
+    console.log("Done");
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });

connectors/src/admin/db.ts

@@ -1,3 +1,5 @@
+import { Sequelize } from "sequelize";
+
 import {
   Connector,
   GithubConnectorState,
@@ -10,6 +12,7 @@ import {
   NotionConnectorState,
   NotionDatabase,
   NotionPage,
+  sequelize_conn,
   SlackChannel,
   SlackChatBotMessage,
   SlackConfiguration,
@@ -33,6 +36,20 @@ async function main(): Promise<void> {
   await GoogleDriveFiles.sync({ alter: true });
   await GoogleDriveSyncToken.sync({ alter: true });
   await GoogleDriveWebhook.sync({ alter: true });
+  await GoogleDriveBFSDedup.sync({ alter: true });
+
+  await addSearchVectorTrigger(
+    sequelize_conn,
+    "notion_pages",
+    "notion_pages_vector_update",
+    "notion_pages_trigger"
+  );
+  await addSearchVectorTrigger(
+    sequelize_conn,
+    "notion_databases",
+    "notion_databases_vector_update",
+    "notion_databases_trigger"
+  );
 }
 
 main()
@@ -49,3 +66,33 @@ main()
     );
     process.exit(1);
   });
+
+async function addSearchVectorTrigger(
+  sequelize_conn: Sequelize,
+  tableName: string,
+  triggerName: string,
+  functionName: string
+) {
+  // this creates/updates a function that will be called on every insert/update
+  await sequelize_conn.query(`
+      CREATE OR REPLACE FUNCTION ${functionName}() RETURNS trigger AS $$
+      begin
+        if TG_OP = 'INSERT' OR new.title IS DISTINCT FROM old.title then
+          new.titleSearchVector := to_tsvector('english', coalesce(new.title, ''));
+        end if;
+        return new;
+      end
+      $$ LANGUAGE plpgsql;
+    `);
+
+  // this creates/updates a trigger that will call the function above
+  await sequelize_conn.query(`
+      DO $$ BEGIN
+        IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname = '${triggerName}') THEN
+          CREATE TRIGGER ${triggerName}
+          BEFORE INSERT OR UPDATE ON "${tableName}"
+          FOR EACH ROW EXECUTE FUNCTION ${functionName}();
+        END IF;
+      END $$;
+    `);
+}

connectors/src/lib/models.ts

@@ -442,6 +442,7 @@ export class NotionPage extends Model<
   declare parentType?: string | null;
   declare parentId?: string | null;
   declare title?: string | null;
+  declare titleSearchVector: unknown;
   declare notionUrl?: string | null;
 
   declare connectorId: ForeignKey<Connector["id"]> | null;
@@ -492,6 +493,10 @@ NotionPage.init(
       type: DataTypes.TEXT,
       allowNull: true,
     },
+    titleSearchVector: {
+      type: DataTypes.TSVECTOR,
+      allowNull: true,
+    },
     notionUrl: {
       type: DataTypes.STRING,
       allowNull: true,
@@ -504,6 +509,11 @@ NotionPage.init(
       { fields: ["connectorId"] },
       { fields: ["lastSeenTs"] },
       { fields: ["parentId"] },
+      {
+        fields: ["titleSearchVector"],
+        using: "gist",
+        name: "notion_pages_title_search_vector_gist_idx",
+      },
     ],
     modelName: "notion_pages",
   }
@@ -526,6 +536,7 @@ export class NotionDatabase extends Model<
   declare parentType?: string | null;
   declare parentId?: string | null;
   declare title?: string | null;
+  declare titleSearchVector: unknown;
   declare notionUrl?: string | null;
 
   declare connectorId: ForeignKey<Connector["id"]> | null;
@@ -572,6 +583,10 @@ NotionDatabase.init(
       type: DataTypes.TEXT,
       allowNull: true,
     },
+    titleSearchVector: {
+      type: DataTypes.TSVECTOR,
+      allowNull: true,
+    },
     notionUrl: {
       type: DataTypes.STRING,
       allowNull: true,
@@ -584,6 +599,11 @@ NotionDatabase.init(
       { fields: ["connectorId", "skipReason"] },
       { fields: ["lastSeenTs"] },
       { fields: ["parentId"] },
+      {
+        fields: ["titleSearchVector"],
+        using: "gist",
+        name: "notion_databases_title_search_vector_gist_idx",
+      },
     ],
     modelName: "notion_databases",
   }