Fix excessive PDF newlines (langchain-ai#3218)

* fix: remove "\n" between words when use pdf-loader * Add fix to web PDF loader, add tests * Comment + log * Update test * Fix directory test --------- Co-authored-by: ppxu <[email protected]>
malandis · Nov 10, 2023 · 65f68b7 · 65f68b7
1 parent 85734d8
commit 65f68b7
Show file tree

Hide file tree

Showing 6 changed files with 63 additions and 10 deletions.
diff --git a/langchain/src/document_loaders/fs/pdf.ts b/langchain/src/document_loaders/fs/pdf.ts
@@ -1,4 +1,3 @@
-import type { TextItem } from "pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js";
 import { Document } from "../../document.js";
 import { BufferLoader } from "./buffer.js";
 import { formatDocumentsAsString } from "../../util/document.js";
@@ -61,9 +60,23 @@ export class PDFLoader extends BufferLoader {
         continue;
       }
 
-      const text = content.items
-        .map((item) => (item as TextItem).str)
-        .join("\n");
+      // Eliminate excessive newlines
+      // Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16
+      let lastY;
+      const textItems = [];
+      for (const item of content.items) {
+        if ("str" in item) {
+          if (lastY === item.transform[5] || !lastY) {
+            textItems.push(item.str);
+          } else {
+            textItems.push(`\n${item.str}`);
+          }
+          // eslint-disable-next-line prefer-destructuring
+          lastY = item.transform[5];
+        }
+      }
+
+      const text = textItems.join(" ");
 
       documents.push(
         new Document({

diff --git a/langchain/src/document_loaders/tests/directory.test.ts b/langchain/src/document_loaders/tests/directory.test.ts
@@ -29,12 +29,13 @@ test("Test Directory loader", async () => {
     UnknownHandling.Ignore
   );
   const docs = await loader.load();
-  expect(docs.length).toBe(122);
+  expect(docs.length).toBe(123);
   expect(docs.map((d) => d.metadata.source).sort()).toEqual([
     // PDF
     ...Array.from({ length: 15 }, (_) =>
       path.resolve(directoryPath, "1706.03762.pdf")
     ),
+    path.resolve(directoryPath, "Jacob_Lee_Resume_2023.pdf"),
     // CSV
     ...Array.from({ length: 32 }, (_) =>
       path.resolve(

diff --git a/langchain/src/document_loaders/tests/example_data/Jacob_Lee_Resume_2023.pdf b/langchain/src/document_loaders/tests/example_data/Jacob_Lee_Resume_2023.pdf
diff --git a/langchain/src/document_loaders/tests/pdf.test.ts b/langchain/src/document_loaders/tests/pdf.test.ts
@@ -26,3 +26,14 @@ test("Test PDF loader from file to single document", async () => {
   expect(docs.length).toBe(1);
   expect(docs[0].pageContent).toContain("Attention Is All You Need");
 });
+
+test("Test PDF loader should not create documents with excessive newlines", async () => {
+  const filePath = path.resolve(
+    path.dirname(url.fileURLToPath(import.meta.url)),
+    "./example_data/Jacob_Lee_Resume_2023.pdf"
+  );
+  const loader = new PDFLoader(filePath, { splitPages: false });
+  const docs = await loader.load();
+  expect(docs.length).toBe(1);
+  expect(docs[0].pageContent.split("\n").length).toBeLessThan(100);
+});
diff --git a/langchain/src/document_loaders/tests/webpdf.int.test.ts b/langchain/src/document_loaders/tests/webpdf.int.test.ts
@@ -94,3 +94,20 @@ test("Test Web PDF loader with custom pdfjs", async () => {
     }
   `);
 });
+
+test("Test Web PDF loader lines", async () => {
+  const filePath = path.resolve(
+    path.dirname(url.fileURLToPath(import.meta.url)),
+    "./example_data/Jacob_Lee_Resume_2023.pdf"
+  );
+  const loader = new WebPDFLoader(
+    new Blob([await fs.readFile(filePath)], {
+      type: "application/pdf",
+    }),
+    { splitPages: false }
+  );
+  const docs = await loader.load();
+
+  expect(docs.length).toBe(1);
+  expect(docs[0].pageContent.split("\n").length).toBeLessThan(100);
+});
diff --git a/langchain/src/document_loaders/web/pdf.ts b/langchain/src/document_loaders/web/pdf.ts
@@ -1,5 +1,3 @@
-import { type TextItem } from "pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js";
-
 import { Document } from "../../document.js";
 import { BaseDocumentLoader } from "../base.js";
 import { formatDocumentsAsString } from "../../util/document.js";
@@ -48,9 +46,22 @@ export class WebPDFLoader extends BaseDocumentLoader {
         continue;
       }
 
-      const text = content.items
-        .map((item) => (item as TextItem).str)
-        .join("\n");
+      // Eliminate excessive newlines
+      // Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16
+      let lastY;
+      const textItems = [];
+      for (const item of content.items) {
+        if ("str" in item) {
+          if (lastY === item.transform[5] || !lastY) {
+            textItems.push(item.str);
+          } else {
+            textItems.push(`\n${item.str}`);
+          }
+          // eslint-disable-next-line prefer-destructuring
+          lastY = item.transform[5];
+        }
+      }
+      const text = textItems.join(" ");
 
       documents.push(
         new Document({