Handle exception when get a empty file (#265)

fsspec · Nov 4, 2024 · 886b7db · 886b7db
1 parent 4a2f861
commit 886b7db
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 21 deletions.
diff --git a/tosfs/core.py b/tosfs/core.py
@@ -55,7 +55,7 @@
 from tosfs.fsspec_utils import glob_translate
 from tosfs.models import DeletingObject
 from tosfs.mpu import MultipartUploader
-from tosfs.retry import retryable_func_executor
+from tosfs.retry import INVALID_RANGE_CODE, retryable_func_executor
 from tosfs.tag import BucketTagMgr
 from tosfs.utils import find_bucket_key, get_brange
 
@@ -1649,16 +1649,25 @@ def _open_remote_file(
     ) -> Tuple[BinaryIO, int]:
         if kwargs.get("callback") is not None:
             kwargs.pop("callback")
-        resp = retryable_func_executor(
-            lambda: self.tos_client.get_object(
-                bucket,
-                key,
-                version_id=version_id,
-                range_start=range_start,
-                **kwargs,
-            ),
-            max_retry_num=self.max_retry_num,
-        )
+        try:
+            resp = retryable_func_executor(
+                lambda: self.tos_client.get_object(
+                    bucket,
+                    key,
+                    version_id=version_id,
+                    range_start=range_start,
+                    **kwargs,
+                ),
+                max_retry_num=self.max_retry_num,
+            )
+        except TosServerError as e:
+            if e.status_code == INVALID_RANGE_CODE:
+                obj_info = self._object_info(bucket=bucket, key=key)
+                if obj_info["size"] == 0 or range_start == obj_info["size"]:
+                    return io.BytesIO(), 0
+            else:
+                raise e
+
         return resp.content, resp.content_length
 
     def _bucket_info(self, bucket: str) -> dict:

diff --git a/tosfs/retry.py b/tosfs/retry.py
@@ -37,6 +37,7 @@
 CONFLICT_CODE = 409
 TOO_MANY_REQUESTS_CODE = 429
 SERVICE_UNAVAILABLE = 503
+INVALID_RANGE_CODE = 416
 
 TOS_SERVER_RETRYABLE_STATUS_CODES = {
     CONFLICT_CODE,

diff --git a/tosfs/tests/test_tosfs.py b/tosfs/tests/test_tosfs.py
@@ -401,19 +401,26 @@ def test_get_file(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -
     file_name = random_str()
     file_content = "hello world"
     rpath = f"{bucket}/{temporary_workspace}/{file_name}"
-    lpath = f"{tempfile.mkdtemp()}/{file_name}"
-    assert not os.path.exists(lpath)
 
-    bucket, key, _ = tosfs._split_path(rpath)
-    with tosfs.open(rpath, "w") as f:
-        f.write(file_content)
+    with tempfile.TemporaryDirectory() as local_temp_dir:
+        tosfs.touch(rpath)
+        tosfs.get_file(rpath, f"{local_temp_dir}/{file_name}")
+        tosfs.rm_file(rpath)
 
-    tosfs.get_file(rpath, lpath)
-    with open(lpath, "r") as f:
-        assert f.read() == file_content
+    with tempfile.TemporaryDirectory() as local_temp_dir:
+        lpath = f"{local_temp_dir}/{file_name}"
+        assert not os.path.exists(lpath)
 
-    with pytest.raises(FileNotFoundError):
-        tosfs.get_file(f"{bucket}/{temporary_workspace}/nonexistent", lpath)
+        bucket, key, _ = tosfs._split_path(rpath)
+        with tosfs.open(rpath, "w") as f:
+            f.write(file_content)
+
+        tosfs.get_file(rpath, lpath)
+        with open(lpath, "r") as f:
+            assert f.read() == file_content
+
+        with pytest.raises(FileNotFoundError):
+            tosfs.get_file(f"{bucket}/{temporary_workspace}/nonexistent", lpath)
 
 
 def test_walk(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: