Flav/add log to webcrawler (#3967)

* Add url in webcrawler error logs * 🔈
dust-tt · Feb 27, 2024 · ada0942 · ada0942
1 parent 14081cf
commit ada0942
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/connectors/src/connectors/webcrawler/temporal/activities.ts b/connectors/src/connectors/webcrawler/temporal/activities.ts
@@ -212,6 +212,7 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
                 configId: webCrawlerConfig.id,
                 documentLen: extracted.length,
                 title: pageTitle,
+                url,
               },
               `Document is empty or too big to be upserted. Skipping`
             );
@@ -224,6 +225,7 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
               error: e,
               connectorId: connector.id,
               configId: webCrawlerConfig.id,
+              url,
             },
             "Webcrawler error while upserting document"
           );