Cleaning up the createConnector Interface

connectors/src/api/configuration.ts

@@ -1,5 +1,6 @@
 import type {
   ConnectorType,
+  UpdateConnectorConfigurationType,
   WithConnectorsAPIErrorReponse,
 } from "@dust-tt/types";
 import {
@@ -20,7 +21,7 @@ const _patchConnectorConfiguration = async (
   req: Request<
     { connector_id: string },
     PatchConnectorConfigurationResBody,
-    { configuration: unknown }
+    UpdateConnectorConfigurationType
   >,
   res: Response<PatchConnectorConfigurationResBody>
 ) => {

connectors/src/api/create_connector.ts

@@ -4,7 +4,9 @@ import type {
 } from "@dust-tt/types";
 import {
   ConnectorCreateRequestBodySchema,
+  ioTsParsePayload,
   isConnectorProvider,
+  WebCrawlerConfigurationTypeSchema,
 } from "@dust-tt/types";
 import type { Request, Response } from "express";
 import { isLeft } from "fp-ts/lib/Either";
@@ -58,15 +60,47 @@ const _createConnectorAPIHandler = async (
     const connectorCreator =
       CREATE_CONNECTOR_BY_TYPE[req.params.connector_provider];
 
-    const connectorRes = await connectorCreator(
-      {
-        workspaceAPIKey: workspaceAPIKey,
-        dataSourceName: dataSourceName,
-        workspaceId: workspaceId,
-      },
-      connectionId,
-      configuration
-    );
+    let connectorRes: Awaited<ReturnType<typeof connectorCreator>> | null =
+      null;
+    switch (req.params.connector_provider) {
+      case "webcrawler": {
+        const configurationRes = ioTsParsePayload(
+          configuration,
+          WebCrawlerConfigurationTypeSchema
+        );
+        if (configurationRes.isErr()) {
+          return apiError(req, res, {
+            status_code: 400,
+            api_error: {
+              type: "invalid_request_error",
+              message: `Invalid request body: ${configurationRes.error}`,
+            },
+          });
+        }
+        connectorRes = await connectorCreator(
+          {
+            workspaceAPIKey: workspaceAPIKey,
+            dataSourceName: dataSourceName,
+            workspaceId: workspaceId,
+          },
+          connectionId,
+          configurationRes.value
+        );
+        break;
+      }
+
+      default:
+        connectorRes = await connectorCreator(
+          {
+            workspaceAPIKey: workspaceAPIKey,
+            dataSourceName: dataSourceName,
+            workspaceId: workspaceId,
+          },
+          connectionId,
+          undefined
+        );
+    }
+
     if (connectorRes.isErr()) {
       return apiError(req, res, {
         status_code: 500,

connectors/src/api/update_connector.ts

@@ -5,7 +5,6 @@ import { isLeft } from "fp-ts/lib/Either";
 import * as reporter from "io-ts-reporters";
 
 import { UPDATE_CONNECTOR_BY_TYPE } from "@connectors/connectors";
-import type {} from "@connectors/connectors/interface";
 import { apiError, withLogging } from "@connectors/logger/withlogging";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
 

connectors/src/connectors/interface.ts

@@ -13,7 +13,7 @@ import type { DataSourceConfig } from "@connectors/types/data_source_config";
 export type ConnectorCreate = (
   DataSourceConfig: DataSourceConfig,
   connectionid: string,
-  configuration?: unknown
+  configuration?: ConnectorConfiguration
 ) => Promise<Result<string, Error>>;
 
 export type ConnectorUpdater = (
@@ -82,10 +82,6 @@ export type ConnectorGarbageCollector = (
   connectorId: ModelId
 ) => Promise<Result<string, Error>>;
 
-export type ConnectorConfigurationGetter = (
-  connectorId: ModelId
-) => Promise<Result<ConnectorConfiguration, Error>>;
-
 export type ConnectorConfigurationSetter = (
   connectorId: ModelId,
   configuration: ConnectorConfiguration

connectors/src/connectors/webcrawler/index.ts

@@ -11,8 +11,6 @@ import {
   Ok,
   WEBCRAWLER_MAX_PAGES,
 } from "@dust-tt/types";
-import { ioTsParsePayload } from "@dust-tt/types";
-import { WebCrawlerConfigurationTypeSchema } from "@dust-tt/types";
 
 import {
   getDisplayNameForPage,
@@ -37,43 +35,34 @@ import {
 export async function createWebcrawlerConnector(
   dataSourceConfig: DataSourceConfig,
   connectionId: string,
-  configuration?: unknown
+  configuration?: WebCrawlerConfigurationType
 ): Promise<Result<string, Error>> {
-  const webCrawlerConfigurationRes = ioTsParsePayload(
-    configuration,
-    WebCrawlerConfigurationTypeSchema
-  );
-  if (webCrawlerConfigurationRes.isErr()) {
-    return new Err(
-      new Error(
-        `Invalid request: ${webCrawlerConfigurationRes.error.join(", ")}`
-      )
-    );
+  if (!configuration) {
+    throw new Error("Configuration is required");
   }
-  const webCrawlerConfiguration = webCrawlerConfigurationRes.value;
-  const depth = webCrawlerConfiguration.depth;
+  const depth = configuration.depth;
   if (!isDepthOption(depth)) {
     return new Err(new Error("Invalid depth option"));
   }
-  if (webCrawlerConfiguration.maxPageToCrawl > WEBCRAWLER_MAX_PAGES) {
+  if (configuration.maxPageToCrawl > WEBCRAWLER_MAX_PAGES) {
     return new Err(
       new Error(`Maximum value for Max Page is ${WEBCRAWLER_MAX_PAGES}`)
     );
   }
 
   const webCrawlerConfigurationBlob = {
-    url: webCrawlerConfiguration.url,
-    maxPageToCrawl: webCrawlerConfiguration.maxPageToCrawl,
-    crawlMode: webCrawlerConfiguration.crawlMode,
+    url: configuration.url,
+    maxPageToCrawl: configuration.maxPageToCrawl,
+    crawlMode: configuration.crawlMode,
     depth: depth,
-    crawlFrequency: webCrawlerConfiguration.crawlFrequency,
+    crawlFrequency: configuration.crawlFrequency,
     lastCrawledAt: null,
   };
 
   const connector = await ConnectorResource.makeNew(
     "webcrawler",
     {
-      connectionId: webCrawlerConfiguration.url,
+      connectionId: configuration.url,
       workspaceAPIKey: dataSourceConfig.workspaceAPIKey,
       workspaceId: dataSourceConfig.workspaceId,
       dataSourceName: dataSourceConfig.dataSourceName,
@@ -362,28 +351,6 @@ export async function retrieveWebCrawlerContentNodeParents(
   return new Ok(parents);
 }
 
-export async function getWebcrawlerConfiguration(
-  connectorId: ModelId
-): Promise<Result<WebCrawlerConfigurationType, Error>> {
-  const connector = await ConnectorResource.fetchById(connectorId);
-  if (!connector) {
-    return new Err(new Error("Connector not found"));
-  }
-  const config = await WebCrawlerConfiguration.findOne({
-    where: { connectorId: connector.id },
-  });
-  if (!config) {
-    return new Err(new Error("Webcrawler configuration not found"));
-  }
-  return new Ok({
-    url: config.url,
-    maxPageToCrawl: config.maxPageToCrawl,
-    crawlMode: config.crawlMode,
-    depth: config.depth,
-    crawlFrequency: config.crawlFrequency,
-  });
-}
-
 export async function setWebcrawlerConfiguration(
   connectorId: ModelId,
   configuration: WebCrawlerConfigurationType

connectors/src/lib/renderers/connector.ts

@@ -38,6 +38,8 @@ async function renderConfiguration(connector: ConnectorResource) {
       }
       return renderWebcrawlerConfiguration(config);
     }
+    default:
+      return undefined;
   }
 }
 

front/pages/api/w/[wId]/data_sources/[name]/configuration.ts

@@ -136,7 +136,7 @@ async function handler(
 
       const updateRes = await connectorsAPI.updateConfiguration({
         connectorId: dataSource.connectorId.toString(),
-        configuration: parseRes.value,
+        configuration: { configuration: parseRes.value.configuration },
       });
       if (updateRes.isErr()) {
         return apiError(

front/pages/api/w/[wId]/data_sources/managed.ts

@@ -143,7 +143,7 @@ async function handler(
               status_code: 400,
               api_error: {
                 type: "invalid_request_error",
-                message: "connectionId is required for OAuth connectors.",
+                message: "connectionId is required.",
               },
             });
           }

types/src/front/lib/connectors_api.ts

@@ -3,7 +3,10 @@ import {
   AdminResponseType,
 } from "../../connectors/admin/cli";
 import { ConnectorsAPIError, isConnectorsAPIError } from "../../connectors/api";
-import { ConnectorConfiguration } from "../../connectors/api_handlers/connector_configuration";
+import {
+  ConnectorConfiguration,
+  UpdateConnectorConfigurationType,
+} from "../../connectors/api_handlers/connector_configuration";
 import { ConnectorCreateRequestBody } from "../../connectors/api_handlers/create_connector";
 import { UpdateConnectorRequestBody } from "../../connectors/api_handlers/update_connector";
 import { ContentNodesViewType } from "../../connectors/content_nodes";
@@ -153,7 +156,7 @@ export class ConnectorsAPI {
     configuration,
   }: {
     connectorId: string;
-    configuration: unknown;
+    configuration: UpdateConnectorConfigurationType;
   }): Promise<ConnectorsAPIResponse<ConnectorType>> {
     const res = await fetch(
       `${CONNECTORS_API}/connectors/${encodeURIComponent(
@@ -162,7 +165,9 @@ export class ConnectorsAPI {
       {
         method: "PATCH",
         headers: this.getDefaultHeaders(),
-        body: JSON.stringify(configuration),
+        body: JSON.stringify(
+          configuration satisfies UpdateConnectorConfigurationType
+        ),
       }
     );