Github Sync updates github docs parents Field

connectors/migrations/20230906_3_github_fill_parents_field.ts

@@ -0,0 +1,104 @@
+import {
+  getDiscussionDocumentId,
+  getIssueDocumentId,
+} from "@connectors/connectors/github/temporal/activities";
+import { updateDocumentParentsField } from "@connectors/lib/data_sources";
+import {
+  Connector,
+  GithubDiscussion,
+  GithubIssue,
+} from "@connectors/lib/models";
+
+async function main() {
+  if (!process.argv[2]) {
+    console.error("Missing workspace id or 'all' as first argument");
+    process.exit(1);
+  }
+  // if first arg is "all", update all connectors, else update only the
+  // connector for the corresponding workspace id
+  const connectors =
+    process.argv[2] === "all"
+      ? await Connector.findAll({
+          where: {
+            type: "github",
+          },
+        })
+      : await Connector.findAll({
+          where: {
+            type: "github",
+            workspaceId: process.argv[2],
+          },
+        });
+
+  for (const connector of connectors) {
+    console.log(`Updating parents field for connector ${connector.id}`);
+    await updateDiscussionsParentsFieldForConnector(connector);
+    await updateIssuesParentsFieldForConnector(connector);
+  }
+}
+
+async function updateDiscussionsParentsFieldForConnector(connector: Connector) {
+  // get all distinct documentIds and their channel ids from slack messages in
+  // this connector
+  const documentData = await GithubDiscussion.findAll({
+    where: {
+      connectorId: connector.id,
+    },
+    attributes: ["repoId", "discussionNumber"],
+  });
+  // update all parents fields for discussions by chunks of 128
+  const chunkSize = 32;
+  for (let i = 0; i < documentData.length; i += chunkSize) {
+    const chunk = documentData.slice(i, i + chunkSize);
+    console.log(`Updating ${chunk.length} documents`);
+    // update parents field for each document of the chunk, in parallel
+    await Promise.all(
+      chunk.map(async (document) => {
+        const docId = getDiscussionDocumentId(
+          document.repoId,
+          document.discussionNumber
+        );
+        await updateDocumentParentsField(connector, docId, [
+          getDiscussionDocumentId(document.repoId, document.discussionNumber),
+          document.repoId,
+        ]);
+      })
+    );
+  }
+}
+
+async function updateIssuesParentsFieldForConnector(connector: Connector) {
+  // get all distinct issues  and their repo ids fro
+  const documentData = await GithubIssue.findAll({
+    where: {
+      connectorId: connector.id,
+    },
+    attributes: ["repoId", "issueNumber"],
+  });
+  // update all parents fields for all issues by chunks of 128
+  const chunkSize = 32;
+  for (let i = 0; i < documentData.length; i += chunkSize) {
+    const chunk = documentData.slice(i, i + chunkSize);
+    console.log(`Updating ${chunk.length} documents`);
+    // update parents field for each document of the chunk, in parallel
+    await Promise.all(
+      chunk.map(async (document) => {
+        const docId = getIssueDocumentId(document.repoId, document.issueNumber);
+        await updateDocumentParentsField(connector, docId, [
+          getIssueDocumentId(document.repoId, document.issueNumber),
+          document.repoId,
+        ]);
+      })
+    );
+  }
+}
+
+main()
+  .then(() => {
+    console.log("Done");
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });

connectors/migrations/20230906_notion_fill_parents_field.ts

@@ -0,0 +1,52 @@
+import { updateAllParentsFields } from "@connectors/connectors/notion/lib/parents";
+import { Connector, NotionDatabase, NotionPage } from "@connectors/lib/models";
+
+async function main() {
+  // if first arg is "all", update all connectors, else update only the
+  // connector for the corresponding workspace id
+  const connectors =
+    process.argv[2] === "all"
+      ? await Connector.findAll({
+          where: {
+            type: "notion",
+          },
+        })
+      : await Connector.findAll({
+          where: {
+            type: "notion",
+            workspaceId: process.argv[2],
+          },
+        });
+
+  for (const connector of connectors) {
+    console.log(`Updating parents field for connector ${connector.id}`);
+    await updateParentsFieldForConnector(connector);
+  }
+}
+
+async function updateParentsFieldForConnector(connector: Connector) {
+  // get all pages and databases for this connector
+  const pages = await NotionPage.findAll({
+    where: {
+      connectorId: connector.id,
+    },
+  });
+  const databases = await NotionDatabase.findAll({
+    where: {
+      connectorId: connector.id,
+    },
+  });
+
+  // update all parents fields for all pages and databases
+  await updateAllParentsFields(connector, [...pages, ...databases]);
+}
+
+main()
+  .then(() => {
+    console.log("Done");
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });

connectors/src/connectors/github/temporal/activities.ts

@@ -154,7 +154,16 @@ export async function githubUpsertIssueActivity(
     documentUrl: issue.url,
     timestampMs: lastUpdateTimestamp,
     tags: tags,
-    parents: [],
+    // The convention for parents is to use the external id string; it is ok for
+    // repos, but not practical for issues since the external id is the
+    // issue number, which is not guaranteed unique in the workspace.
+    // Therefore as a special case we use getIssueDocumentId() to get a parent string
+    // The repo id from github is globally unique so used as-is, as per
+    // convention to use the external id string.
+    parents: [
+      getIssueDocumentId(repoId.toString(), issue.number),
+      repoId.toString(),
+    ],
     retries: 3,
     delayBetweenRetriesMs: 500,
     loggerArgs: { ...loggerArgs, provider: "github" },
@@ -281,7 +290,16 @@ export async function githubUpsertDiscussionActivity(
     documentUrl: discussion.url,
     timestampMs: new Date(discussion.createdAt).getTime(),
     tags,
-    parents: [],
+    // The convention for parents is to use the external id string; it is ok for
+    // repos, but not practical for discussions since the external id is the
+    // issue number, which is not guaranteed unique in the workspace. Therefore
+    // as a special case we use getDiscussionDocumentId() to get a parent string
+    // The repo id from github is globally unique so used as-is, as per
+    // convention to use the external id string.
+    parents: [
+      getDiscussionDocumentId(repoId.toString(), discussionNumber),
+      repoId.toString(),
+    ],
     retries: 3,
     delayBetweenRetriesMs: 500,
     loggerArgs: { ...loggerArgs, provider: "github" },
@@ -585,11 +603,14 @@ function renderGithubUser(user: GithubUser | null): string {
   return `@${user.id}`;
 }
 
-function getIssueDocumentId(repoId: string, issueNumber: number): string {
+export function getIssueDocumentId(
+  repoId: string,
+  issueNumber: number
+): string {
   return `github-issue-${repoId}-${issueNumber}`;
 }
 
-function getDiscussionDocumentId(
+export function getDiscussionDocumentId(
   repoId: string,
   discussionNumber: number
 ): string {

connectors/src/connectors/notion/lib/connectors_db_helpers.ts

@@ -218,3 +218,59 @@ export async function getNotionDatabaseFromConnectorsDb(
 
   return NotionDatabase.findOne({ where });
 }
+
+/**
+ * Get children *that are pages* of a given notion page or database
+ *
+ * !! Not children *of a page*
+ */
+export async function getPageChildrenOf(
+  dataSourceInfo: DataSourceInfo,
+  notionId: string
+): Promise<NotionPage[]> {
+  const connector = await Connector.findOne({
+    where: {
+      type: "notion",
+      workspaceId: dataSourceInfo.workspaceId,
+      dataSourceName: dataSourceInfo.dataSourceName,
+    },
+  });
+  if (!connector) {
+    throw new Error("Could not find connector");
+  }
+
+  return NotionPage.findAll({
+    where: {
+      parentId: notionId,
+      connectorId: connector.id,
+    },
+  });
+}
+
+/**
+ * Get children *that are databases* of a given notion page or database
+ *
+ * !! Not children *of a database*
+ */
+export async function getDatabaseChildrenOf(
+  dataSourceInfo: DataSourceInfo,
+  notionId: string
+): Promise<NotionDatabase[]> {
+  const connector = await Connector.findOne({
+    where: {
+      type: "notion",
+      workspaceId: dataSourceInfo.workspaceId,
+      dataSourceName: dataSourceInfo.dataSourceName,
+    },
+  });
+  if (!connector) {
+    throw new Error("Could not find connector");
+  }
+
+  return NotionDatabase.findAll({
+    where: {
+      parentId: notionId,
+      connectorId: connector.id,
+    },
+  });
+}