Performance: fetch range optimize

fsspec · Sep 23, 2024 · 3b7ee18 · 3b7ee18
1 parent f4a634c
commit 3b7ee18
Showing 1 changed file with 11 additions and 8 deletions.
diff --git a/tosfs/core.py b/tosfs/core.py
@@ -2091,7 +2091,7 @@ def _upload_chunk(self, final: bool = False) -> bool:
 
     def _fetch_range(self, start: int, end: int) -> bytes:
         if start == end:
-            logger.debug(
+            logger.warning(
                 "skip fetch for negative range - bucket=%s,key=%s,start=%d,end=%d",
                 self.bucket,
                 self.key,
@@ -2102,13 +2102,16 @@ def _fetch_range(self, start: int, end: int) -> bytes:
         logger.debug("Fetch: %s/%s, %s-%s", self.bucket, self.key, start, end)
 
         def fetch() -> bytes:
-            temp_buffer = io.BytesIO()
-            for chunk in self.fs.tos_client.get_object(
-                self.bucket, self.key, self.version_id, range_start=start, range_end=end
-            ):
-                temp_buffer.write(chunk)
-            temp_buffer.seek(0)
-            return temp_buffer.read()
+            with io.BytesIO() as temp_buffer:
+                for chunk in self.fs.tos_client.get_object(
+                    self.bucket,
+                    self.key,
+                    self.version_id,
+                    range_start=start,
+                    range_end=end,
+                ):
+                    temp_buffer.write(chunk)
+                return temp_buffer.getvalue()
 
         return retryable_func_executor(fetch, max_retry_num=self.fs.max_retry_num)