feat(connectors): add endpoints for resources titles and parents (#1557)

* feat(connectors): add endpoint to return resources detail

* plumbing

* switch to post + add front binding

connectors/src/api/get_resources_parents.ts

@@ -0,0 +1,92 @@
+import { Request, Response } from "express";
+import { zip } from "fp-ts/lib/Array";
+import { isLeft } from "fp-ts/lib/Either";
+import * as t from "io-ts";
+import * as reporter from "io-ts-reporters";
+
+import { RETRIEVE_RESOURCE_PARENTS_BY_TYPE } from "@connectors/connectors";
+import { Connector } from "@connectors/lib/models";
+import logger from "@connectors/logger/logger";
+import { apiError, withLogging } from "@connectors/logger/withlogging";
+
+const GetResourcesParentsRequestBodySchema = t.type({
+  resourceInternalIds: t.array(t.string),
+});
+
+export type GetResourcesParentsRequestBody = t.TypeOf<
+  typeof GetResourcesParentsRequestBodySchema
+>;
+
+type GetResourcesParentsResponseBody = {
+  resources: {
+    internalId: string;
+    parents: string[] | null;
+  }[];
+};
+
+const _getResourcesParents = async (
+  req: Request<
+    { connector_id: string },
+    GetResourcesParentsResponseBody,
+    GetResourcesParentsRequestBody
+  >,
+  res: Response<GetResourcesParentsResponseBody>
+) => {
+  const connector = await Connector.findByPk(req.params.connector_id);
+  if (!connector) {
+    return apiError(req, res, {
+      api_error: {
+        type: "connector_not_found",
+        message: "Connector not found",
+      },
+      status_code: 404,
+    });
+  }
+
+  const bodyValidation = GetResourcesParentsRequestBodySchema.decode(req.body);
+  if (isLeft(bodyValidation)) {
+    const pathError = reporter.formatValidationErrors(bodyValidation.left);
+    return apiError(req, res, {
+      api_error: {
+        type: "invalid_request_error",
+        message: `Invalid request body: ${pathError}`,
+      },
+      status_code: 400,
+    });
+  }
+
+  const { resourceInternalIds } = bodyValidation.right;
+
+  const parentsGetter = RETRIEVE_RESOURCE_PARENTS_BY_TYPE[connector.type];
+  const parentsResults = await Promise.all(
+    resourceInternalIds.map((resourceInternalId) =>
+      parentsGetter(connector.id, resourceInternalId)
+    )
+  );
+  const resources: { internalId: string; parents: string[] }[] = [];
+
+  for (const [internalId, parentsResult] of zip(
+    resourceInternalIds,
+    parentsResults
+  )) {
+    if (parentsResult.isErr()) {
+      logger.error(parentsResult.error, "Failed to get resource parents");
+      return apiError(req, res, {
+        api_error: {
+          type: "internal_server_error",
+          message: parentsResult.error.message,
+        },
+        status_code: 500,
+      });
+    }
+
+    resources.push({
+      internalId,
+      parents: parentsResult.value,
+    });
+  }
+
+  return res.status(200).json({ resources });
+};
+
+export const getResourcesParentsAPIHandler = withLogging(_getResourcesParents);

connectors/src/api/get_resources_titles.ts

@@ -0,0 +1,86 @@
+import { Request, Response } from "express";
+import { isLeft } from "fp-ts/lib/Either";
+import * as t from "io-ts";
+import * as reporter from "io-ts-reporters";
+
+import { BATCH_RETRIEVE_RESOURCE_TITLE_BY_TYPE } from "@connectors/connectors";
+import { Connector } from "@connectors/lib/models";
+import { Result } from "@connectors/lib/result";
+import { apiError, withLogging } from "@connectors/logger/withlogging";
+
+const GetResourcesTitlesRequestBodySchema = t.type({
+  resourceInternalIds: t.array(t.string),
+});
+type GetResourcesTitlesRequestBody = t.TypeOf<
+  typeof GetResourcesTitlesRequestBodySchema
+>;
+
+type GetResourcesTitlesResponseBody = {
+  resources: {
+    internalId: string;
+    title: string | null;
+  }[];
+};
+
+const _getResourcesTitles = async (
+  req: Request<
+    { connector_id: string },
+    GetResourcesTitlesResponseBody,
+    GetResourcesTitlesRequestBody
+  >,
+  res: Response<GetResourcesTitlesResponseBody>
+) => {
+  const connector = await Connector.findByPk(req.params.connector_id);
+  if (!connector) {
+    return apiError(req, res, {
+      api_error: {
+        type: "connector_not_found",
+        message: "Connector not found",
+      },
+      status_code: 404,
+    });
+  }
+
+  const bodyValidation = GetResourcesTitlesRequestBodySchema.decode(req.body);
+  if (isLeft(bodyValidation)) {
+    const pathError = reporter.formatValidationErrors(bodyValidation.left);
+    return apiError(req, res, {
+      api_error: {
+        type: "invalid_request_error",
+        message: `Invalid request body: ${pathError}`,
+      },
+      status_code: 400,
+    });
+  }
+
+  const { resourceInternalIds } = bodyValidation.right;
+
+  const titlesRes: Result<
+    Record<string, string | null>,
+    Error
+  > = await BATCH_RETRIEVE_RESOURCE_TITLE_BY_TYPE[connector.type](
+    connector.id,
+    resourceInternalIds
+  );
+
+  if (titlesRes.isErr()) {
+    return apiError(req, res, {
+      api_error: {
+        type: "internal_server_error",
+        message: titlesRes.error.message,
+      },
+      status_code: 500,
+    });
+  }
+
+  const titles = titlesRes.value;
+
+  return res.status(200).json({
+    resources: resourceInternalIds.map((internalId) => ({
+      internalId,
+      title: titles[internalId] || null,
+    })),
+  });
+};
+
+export const getResourcesTitlesAPIHandler = withLogging(_getResourcesTitles);

connectors/src/api_server.ts

@@ -20,6 +20,9 @@ import { webhookSlackAPIHandler } from "@connectors/api/webhooks/webhook_slack";
 import logger from "@connectors/logger/logger";
 import { authMiddleware } from "@connectors/middleware/auth";
 
+import { getResourcesParentsAPIHandler } from "./api/get_resources_parents";
+import { getResourcesTitlesAPIHandler } from "./api/get_resources_titles";
+
 export function startServer(port: number) {
   const app = express();
 
@@ -53,6 +56,16 @@ export function startServer(port: number) {
     "/connectors/:connector_id/permissions",
     getConnectorPermissionsAPIHandler
   );
+  app.post(
+    // must be POST because of body
+    "/connectors/:connector_id/resources/parents",
+    getResourcesParentsAPIHandler
+  );
+  app.post(
+    // must be POST because of body
+    "/connectors/:connector_id/resources/titles",
+    getResourcesTitlesAPIHandler
+  );
   app.post(
     "/connectors/:connector_id/permissions",
     setConnectorPermissionsAPIHandler

connectors/src/connectors/github/index.ts

@@ -311,3 +311,42 @@ export async function retrieveGithubConnectorPermissions({
 
   return new Ok(resources);
 }
+
+export async function retrieveGithubReposTitles(
+  connectorId: ModelId,
+  repoIds: string[]
+): Promise<Result<Record<string, string>, Error>> {
+  const c = await Connector.findOne({
+    where: {
+      id: connectorId,
+    },
+  });
+  if (!c) {
+    logger.error({ connectorId }, "Connector not found");
+    return new Err(new Error("Connector not found"));
+  }
+
+  const githubInstallationId = c.connectionId;
+
+  const repoIdsSet = new Set(repoIds);
+
+  // for github, we just fetch all the repos from the github API and only filter the ones we need
+  // this is fine as we don't expect to have a lot of repos (it should rarely be more than 1 api call)
+  const repoTitles: Record<string, string> = {};
+  let pageNumber = 1; // 1-indexed
+  for (;;) {
+    const page = await getReposPage(githubInstallationId, pageNumber);
+    pageNumber += 1;
+    if (page.length === 0) {
+      break;
+    }
+
+    page.forEach((repo) => {
+      if (repoIdsSet.has(repo.id.toString())) {
+        repoTitles[repo.id.toString()] = repo.name;
+      }
+    });
+  }
+
+  return new Ok(repoTitles);
+}

connectors/src/connectors/google_drive/index.ts

@@ -3,7 +3,10 @@ import { drive_v3 } from "googleapis";
 import { GaxiosResponse } from "googleapis-common";
 import { Transaction } from "sequelize";
 
-import { registerWebhook } from "@connectors/connectors/google_drive/lib";
+import {
+  getLocalParents,
+  registerWebhook,
+} from "@connectors/connectors/google_drive/lib";
 import { ConnectorPermissionRetriever } from "@connectors/connectors/interface";
 import {
   Connector,
@@ -36,6 +39,7 @@ import {
 } from "./temporal/activities";
 import { launchGoogleDriveFullSyncWorkflow } from "./temporal/client";
 export type NangoConnectionId = string;
+import { v4 as uuidv4 } from "uuid";
 
 const {
   NANGO_GOOGLE_DRIVE_CONNECTOR_ID,
@@ -519,3 +523,36 @@ export async function setGoogleDriveConnectorPermissions(
 
   return new Ok(undefined);
 }
+
+export async function retrieveGoogleDriveObjectsTitles(
+  connectorId: ModelId,
+  internalIds: string[]
+): Promise<Result<Record<string, string>, Error>> {
+  const googleDriveFiles = await GoogleDriveFiles.findAll({
+    where: {
+      connectorId: connectorId,
+      driveFileId: internalIds,
+    },
+  });
+
+  const titles = googleDriveFiles.reduce((acc, curr) => {
+    acc[curr.driveFileId] = curr.name;
+    return acc;
+  }, {} as Record<string, string>);
+
+  return new Ok(titles);
+}
+
+export async function retrieveGoogleDriveObjectsParents(
+  connectorId: ModelId,
+  internalId: string,
+  memoizationKey?: string
+): Promise<Result<string[], Error>> {
+  const memo = memoizationKey || uuidv4();
+  try {
+    const parents = await getLocalParents(connectorId, internalId, memo);
+    return new Ok(parents);
+  } catch (err) {
+    return new Err(err as Error);
+  }
+}

connectors/src/connectors/google_drive/lib.ts

@@ -2,7 +2,7 @@ import memoize from "lodash.memoize";
 import { v4 as uuidv4 } from "uuid";
 
 import { HTTPError } from "@connectors/lib/error";
-import { GoogleDriveFiles } from "@connectors/lib/models";
+import { GoogleDriveFiles, ModelId } from "@connectors/lib/models";
 import { Err, Ok, type Result } from "@connectors/lib/result.js";
 
 import { getAuthObject } from "./temporal/activities";
@@ -55,7 +55,7 @@ export async function registerWebhook(
 }
 
 async function _getLocalParents(
-  connectorId: string,
+  connectorId: ModelId,
   driveObjectId: string,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars -- used for memoization
   memoizationKey: string

connectors/src/connectors/index.ts

@@ -4,24 +4,29 @@ import {
   fullResyncGithubConnector,
   resumeGithubConnector,
   retrieveGithubConnectorPermissions,
+  retrieveGithubReposTitles,
   stopGithubConnector,
   updateGithubConnector,
 } from "@connectors/connectors/github";
 import {
   cleanupGoogleDriveConnector,
   createGoogleDriveConnector,
   retrieveGoogleDriveConnectorPermissions,
+  retrieveGoogleDriveObjectsParents,
+  retrieveGoogleDriveObjectsTitles,
   setGoogleDriveConnectorPermissions,
   updateGoogleDriveConnector,
 } from "@connectors/connectors/google_drive";
 import { launchGoogleDriveFullSyncWorkflow } from "@connectors/connectors/google_drive/temporal/client";
 import {
   BotEnabledGetter,
   BotToggler,
+  ConnectorBatchResourceTitleRetriever,
   ConnectorCleaner,
   ConnectorCreator,
   ConnectorPermissionRetriever,
   ConnectorPermissionSetter,
+  ConnectorResourceParentsRetriever,
   ConnectorResumer,
   ConnectorStopper,
   ConnectorUpdater,
@@ -33,12 +38,15 @@ import {
   fullResyncNotionConnector,
   resumeNotionConnector,
   retrieveNotionConnectorPermissions,
+  retrieveNotionResourceParents,
+  retrieveNotionResourcesTitles,
   stopNotionConnector,
   updateNotionConnector,
 } from "@connectors/connectors/notion";
 import {
   cleanupSlackConnector,
   createSlackConnector,
+  retrieveSlackChannelsTitles,
   retrieveSlackConnectorPermissions,
   setSlackConnectorPermissions,
   updateSlackConnector,
@@ -184,3 +192,23 @@ export const SET_CONNECTOR_PERMISSIONS_BY_TYPE: Record<
   },
   google_drive: setGoogleDriveConnectorPermissions,
 };
+
+export const BATCH_RETRIEVE_RESOURCE_TITLE_BY_TYPE: Record<
+  ConnectorProvider,
+  ConnectorBatchResourceTitleRetriever
+> = {
+  slack: retrieveSlackChannelsTitles,
+  notion: retrieveNotionResourcesTitles,
+  github: retrieveGithubReposTitles,
+  google_drive: retrieveGoogleDriveObjectsTitles,
+};
+
+export const RETRIEVE_RESOURCE_PARENTS_BY_TYPE: Record<
+  ConnectorProvider,
+  ConnectorResourceParentsRetriever
+> = {
+  notion: retrieveNotionResourceParents,
+  google_drive: retrieveGoogleDriveObjectsParents,
+  slack: async () => new Ok([]), // Slack is flat
+  github: async () => new Ok([]), // Github is flat,
+};