Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactoring PDF loaders: 02 PyMuPDF #29063

Open
wants to merge 45 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
21759e2
Prepare the integration of new versions of PDFLoader.
pprados Jan 2, 2025
4607354
Fix Line too long
pprados Jan 7, 2025
668dc9c
Fix Line too long
pprados Jan 7, 2025
7a5b5c5
Fix Line too long
pprados Jan 7, 2025
6340ded
Fix Line too long
pprados Jan 7, 2025
4845781
Update PyMuPDF
pprados Jan 2, 2025
3beda82
Fix tu
pprados Jan 7, 2025
743a83e
Fix review - step 1
pprados Jan 9, 2025
b623750
Fix all remarques
pprados Jan 10, 2025
20f5a41
Merge remote-tracking branch 'upstream/master' into pprados/02-pymupdf
pprados Jan 10, 2025
91234f0
Fix remarques
pprados Jan 10, 2025
80ee3f7
Fix Images
pprados Jan 13, 2025
66f97cf
Merge remote-tracking branch 'upstream/master' into pprados/02-pymupdf
pprados Jan 13, 2025
0e6c904
Fix Images
pprados Jan 13, 2025
9b45bd8
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 13, 2025
acf4358
Fix deprecated load() with kwargs
pprados Jan 14, 2025
d7d3021
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 14, 2025
4762fab
Change the format for images parser
pprados Jan 14, 2025
6121005
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 14, 2025
5910f99
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 14, 2025
7fc01f3
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 15, 2025
0f654a1
Add format "html" and "markdown" for LLMImageBlobParser
pprados Jan 15, 2025
e4f36ed
Merge remote-tracking branch 'origin/pprados/02-pymupdf' into pprados…
pprados Jan 15, 2025
4a62529
Fix
pprados Jan 15, 2025
1c78325
Bugfix
pprados Jan 15, 2025
1227dbb
Bugfix
pprados Jan 15, 2025
90085e4
Bug fix
pprados Jan 15, 2025
14264e9
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 15, 2025
feacf69
Replace markdown-link to markdown-img
pprados Jan 16, 2025
c074729
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 16, 2025
ee4784d
Update PyMuPDF
pprados Jan 16, 2025
5d4a256
Add default value for properties
pprados Jan 16, 2025
3d15d39
Fix one bug, update some typos, and style doc strings while reading
eyurtsev Jan 16, 2025
0be6c88
Change the strategy for images_inner_format.
pprados Jan 17, 2025
d104ee7
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 17, 2025
023ba11
Fix PIL dependencies
pprados Jan 17, 2025
23a73a9
Fix notebook
pprados Jan 17, 2025
a4587f0
Optimise tests
pprados Jan 17, 2025
d332958
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 17, 2025
4b37b34
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 17, 2025
2281d05
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 17, 2025
0da73f1
Remove Image.__init__
pprados Jan 18, 2025
d012d60
Merge remote-tracking branch 'origin/pprados/02-pymupdf' into pprados…
pprados Jan 18, 2025
882c90d
Merge branch 'master' into pprados/02-pymupdf
pprados Jan 18, 2025
74d3617
Remove Image.__init__
pprados Jan 18, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix Line too long
  • Loading branch information
pprados committed Jan 7, 2025
commit 7a5b5c5c5c975f4eb4ac654747f97a187df973cb
17 changes: 10 additions & 7 deletions libs/community/langchain_community/document_loaders/pdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ class PDFMinerLoader(BasePDFLoader):

def __init__(
self,
file_path: str,
file_path: Union[str, PurePath],
*,
headers: Optional[dict] = None,
extract_images: bool = False,
Expand Down Expand Up @@ -376,7 +376,10 @@ def lazy_load(
class PDFMinerPDFasHTMLLoader(BasePDFLoader):
"""Load `PDF` files as HTML content using `PDFMiner`."""

def __init__(self, file_path: str, *, headers: Optional[dict] = None):
def __init__(self,
file_path: Union[str, PurePath],
*,
headers: Optional[dict] = None):
"""Initialize with a file path."""
try:
from pdfminer.high_level import extract_text_to_fp # noqa:F401
Expand Down Expand Up @@ -414,7 +417,7 @@ class PyMuPDFLoader(BasePDFLoader):

def __init__(
self,
file_path: str,
file_path: Union[str, PurePath],
*,
headers: Optional[dict] = None,
extract_images: bool = False,
Expand Down Expand Up @@ -611,7 +614,7 @@ class PDFPlumberLoader(BasePDFLoader):

def __init__(
self,
file_path: str,
file_path: Union[str, PurePath],
text_kwargs: Optional[Mapping[str, Any]] = None,
dedupe: bool = False,
headers: Optional[dict] = None,
Expand Down Expand Up @@ -890,7 +893,7 @@ def _make_config(self) -> dict:
from dedoc.utils.langchain import make_manager_pdf_config

return make_manager_pdf_config(
file_path=self.file_path,
file_path=str(self.file_path),
parsing_params=self.parsing_parameters,
split=self.split,
)
Expand All @@ -901,7 +904,7 @@ class DocumentIntelligenceLoader(BasePDFLoader):

def __init__(
self,
file_path: str,
file_path: Union[str, PurePath],
client: Any,
model: str = "prebuilt-document",
headers: Optional[dict] = None,
Expand Down Expand Up @@ -1008,7 +1011,7 @@ def lazy_load(self) -> Iterator[Document]:

# Directly call asyncio.run to execute zerox synchronously
zerox_output = asyncio.run(
zerox(file_path=self.file_path, model=self.model, **self.zerox_kwargs)
zerox(file_path=str(self.file_path), model=self.model, **self.zerox_kwargs)
)

# Convert zerox output to Document instances and yield them
Expand Down