bug fix

converting pathlib obj to string
joey00072 · Jun 10, 2024 · d5f6dd3 · d5f6dd3
1 parent 16941b3
commit d5f6dd3
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/ohara/dataset.py b/ohara/dataset.py
@@ -58,7 +58,7 @@ def __init__(
             fpath = str(
                 f"{self.dataset_name.replace('/','-')}--{self.tokenizer.name_or_path.replace('/','-')}"
             )
-            fpath = path.joinpath(fpath).joinpath(split)
+            fpath = str(path.joinpath(fpath).joinpath(split))
 
         self.ds = load_from_disk(fpath)
         self.toks_cycle = cycle(self.ds)

diff --git a/ohara/pretokenize.py b/ohara/pretokenize.py
@@ -74,7 +74,8 @@ def save_pre_tokenized_dataset(self, dataset, split):
         fpath = str(
             f"{self.dataset_name.replace('/','-')}--{self.tokenizer.name_or_path.replace('/','-')}"
         )
-        dataset.save_to_disk(self.output_dir.joinpath(fpath).joinpath(split))
+        fpath = str(self.output_dir.joinpath(fpath).joinpath(split))
+        dataset.save_to_disk(fpath)
 
         print(f"Dataset saved to {self.output_dir}")