Skip to content

Commit

Permalink
Let notion loader support utf-8 and make it default.
Browse files Browse the repository at this point in the history
  • Loading branch information
richarddwang committed Sep 15, 2023
1 parent f9f1340 commit 33a3db6
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions libs/langchain/langchain/document_loaders/notion.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,17 @@
class NotionDirectoryLoader(BaseLoader):
"""Load `Notion directory` dump."""

def __init__(self, path: str):
def __init__(self, path: str, encoding: str = 'utf-8'):
"""Initialize with a file path."""
self.file_path = path
self.encoding = encoding

def load(self) -> List[Document]:
"""Load documents."""
ps = list(Path(self.file_path).glob("**/*.md"))
docs = []
for p in ps:
with open(p) as f:
with open(p, encoding=self.encoding) as f:
text = f.read()
metadata = {"source": str(p)}
docs.append(Document(page_content=text, metadata=metadata))
Expand Down

0 comments on commit 33a3db6

Please sign in to comment.