Bug: Fix rm api and walk api bug

fsspec · Sep 25, 2024 · 9fcc2ca · 9fcc2ca
1 parent 80f71f3
commit 9fcc2ca
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 22 deletions.
diff --git a/tosfs/core.py b/tosfs/core.py
@@ -1837,7 +1837,16 @@ def exists(self, path: str, **kwargs: Any) -> bool:
                     )
                 except TosServerError as ex:
                     if e.status_code == TOS_SERVER_STATUS_CODE_NOT_FOUND:
-                        return False
+                        resp = retryable_func_executor(
+                            lambda: self.tos_client.list_objects_type2(
+                                bucket,
+                                key.rstrip("/") + "/",
+                                start_after=key.rstrip("/") + "/",
+                                max_keys=1,
+                            ),
+                            max_retry_num=self.max_retry_num,
+                        )
+                        return len(resp.contents) > 0
                     else:
                         raise ex
             else:

diff --git a/tosfs/tests/test_tosfs.py b/tosfs/tests/test_tosfs.py
@@ -14,16 +14,13 @@
 import os.path
 import tempfile
 
-import fsspec
 import pytest
 from tos.exceptions import TosServerError
 
 from tosfs.core import TosFileSystem
 from tosfs.exceptions import TosfsError
 from tosfs.utils import create_temp_dir, random_str
 
-fsspec_version = fsspec.__version__
-
 
 def test_ls_bucket(tosfs: TosFileSystem, bucket: str) -> None:
     assert bucket in tosfs.ls("", detail=False)
@@ -226,6 +223,11 @@ def test_exists_object(
     tosfs.rm_file(f"{bucket}/{temporary_workspace}/{file_name}")
     assert not tosfs.exists(f"{bucket}/{temporary_workspace}/{file_name}")
 
+    tosfs.touch(f"{bucket}/{temporary_workspace}/a/b/c/d.txt")
+    assert tosfs.exists(f"{bucket}/{temporary_workspace}/a")
+    assert tosfs.exists(f"{bucket}/{temporary_workspace}/a/b")
+    assert tosfs.exists(f"{bucket}/{temporary_workspace}/a/b/c")
+
 
 def test_mkdir(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None:
     dir_name = random_str()
@@ -635,24 +637,15 @@ def test_glob(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> No
     )
 
     # Test with maxdepth
-    assert (
-        sorted(tosfs.glob(f"{bucket}/{temporary_workspace}/**", maxdepth=2))
-        == sorted(
-            [
-                f"{bucket}/{temporary_workspace}/{dir_name}",
-                f"{bucket}/{temporary_workspace}/{dir_name}/{file_name}",
-                f"{bucket}/{temporary_workspace}/{dir_name}/{sub_dir_name}",
-            ]
-        )
-        if fsspec_version == "2023.5.0"
-        else sorted(
-            [
-                f"{bucket}/{temporary_workspace}",
-                f"{bucket}/{temporary_workspace}/{dir_name}",
-                f"{bucket}/{temporary_workspace}/{dir_name}/{file_name}",
-                f"{bucket}/{temporary_workspace}/{dir_name}/{sub_dir_name}",
-            ]
-        )
+    assert sorted(
+        tosfs.glob(f"{bucket}/{temporary_workspace}/**", maxdepth=2)
+    ) == sorted(
+        [
+            f"{bucket}/{temporary_workspace}",
+            f"{bucket}/{temporary_workspace}/{dir_name}",
+            f"{bucket}/{temporary_workspace}/{dir_name}/{file_name}",
+            f"{bucket}/{temporary_workspace}/{dir_name}/{sub_dir_name}",
+        ]
     )
 
     # Test with detail