From ada09427695a7054519d4b43a8e92bbf4f40537b Mon Sep 17 00:00:00 2001
From: Flavien David <flavien.david74@gmail.com>
Date: Tue, 27 Feb 2024 10:41:45 +0100
Subject: [PATCH] Flav/add log to webcrawler (#3967)
* Add url in webcrawler error logs
* :speaker:
---
connectors/src/connectors/webcrawler/temporal/activities.ts | 2 ++
1 file changed, 2 insertions(+)
diff --git a/connectors/src/connectors/webcrawler/temporal/activities.ts b/connectors/src/connectors/webcrawler/temporal/activities.ts
index 44eee5755bdb..8991df47aa9a 100644
--- a/connectors/src/connectors/webcrawler/temporal/activities.ts
+++ b/connectors/src/connectors/webcrawler/temporal/activities.ts
@@ -212,6 +212,7 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
configId: webCrawlerConfig.id,
documentLen: extracted.length,
title: pageTitle,
+ url,
},
`Document is empty or too big to be upserted. Skipping`
);
@@ -224,6 +225,7 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
error: e,
connectorId: connector.id,
configId: webCrawlerConfig.id,
+ url,
},
"Webcrawler error while upserting document"
);