Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Core: Refactor exception writing method #74

Merged
merged 1 commit into from
Sep 11, 2024
+25 −49
Merged

Core: Refactor exception writing method #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 23 additions & 47 deletions tosfs/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,9 +388,7 @@ def rmdir(self, path: str) -> None:

try:
self.tos_client.delete_object(bucket, key.rstrip("/") + "/")
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except (TosClientError, TosServerError) as e:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e
Expand Down Expand Up @@ -473,11 +471,7 @@ def mkdir(self, path: str, create_parents: bool = True, **kwargs: Any) -> None:
)
else:
self.tos_client.put_object(bucket, key.rstrip("/") + "/")
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
raise e
except FileNotFoundError as e:
except (TosClientError, TosServerError, FileNotFoundError) as e:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e
Expand Down Expand Up @@ -542,9 +536,7 @@ def touch(self, path: str, truncate: bool = True, **kwargs: Any) -> None:

try:
self.tos_client.put_object(bucket, key)
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except (TosClientError, TosServerError) as e:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e
Expand Down Expand Up @@ -587,9 +579,9 @@ def isdir(self, path: str) -> bool:
try:
self.tos_client.head_object(bucket, key)
return True
except tos.exceptions.TosClientError as e:
except TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except TosServerError as e:
if e.status_code == TOS_SERVER_RESPONSE_CODE_NOT_FOUND:
return False
else:
Expand Down Expand Up @@ -622,9 +614,9 @@ def isfile(self, path: str) -> bool:
# Attempt to get the object metadata
self.tos_client.head_object(bucket, key)
return True
except tos.exceptions.TosClientError as e:
except TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except TosServerError as e:
if e.status_code == TOS_SERVER_RESPONSE_CODE_NOT_FOUND:
return False
raise e
Expand Down Expand Up @@ -712,9 +704,7 @@ def put_file(
self.tos_client.complete_multipart_upload(
bucket, key, mpu.upload_id, complete_all=True
)
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except (TosClientError, TosServerError) as e:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e
Expand Down Expand Up @@ -759,7 +749,7 @@ def _read_chunks(body: BinaryIO, f: BinaryIO) -> None:
while True:
try:
chunk = body.read(GET_OBJECT_OPERATION_DEFAULT_READ_CHUNK_SIZE)
except tos.exceptions.TosClientError as e:
except TosClientError as e:
failed_reads += 1
if failed_reads >= RETRY_NUM:
raise e
Expand Down Expand Up @@ -1159,9 +1149,7 @@ def _copy_basic(self, path1: str, path2: str, **kwargs: Any) -> None:
src_key=key1,
src_version_id=ver1,
)
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except (TosClientError, TosServerError) as e:
raise e
except Exception as e:
raise TosfsError("Copy failed (%r -> %r): %s" % (path1, path2, e)) from e
Expand Down Expand Up @@ -1320,9 +1308,7 @@ def _open_remote_file(
**kwargs,
)
return resp.content, resp.content_length
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except (TosClientError, TosServerError) as e:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e
Expand Down Expand Up @@ -1361,9 +1347,9 @@ def _bucket_info(self, bucket: str) -> dict:
try:
self.tos_client.head_bucket(bucket)
return self._fill_bucket_info(bucket)
except tos.exceptions.TosClientError as e:
except TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except TosServerError as e:
if e.status_code == TOS_SERVER_RESPONSE_CODE_NOT_FOUND:
raise FileNotFoundError(bucket) from e
else:
Expand Down Expand Up @@ -1420,9 +1406,9 @@ def _object_info(
"VersionId": out.version_id or "",
"ContentType": out.content_type or "",
}
except tos.exceptions.TosClientError as e:
except TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except TosServerError as e:
if e.status_code == TOS_SERVER_RESPONSE_CODE_NOT_FOUND:
pass
else:
Expand Down Expand Up @@ -1452,11 +1438,7 @@ def _try_dir_info(self, bucket: str, key: str, path: str, fullpath: str) -> dict
}

raise FileNotFoundError(path)
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
raise e
except FileNotFoundError as e:
except (TosClientError, TosServerError, FileNotFoundError) as e:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e
Expand Down Expand Up @@ -1542,9 +1524,9 @@ def _exists_bucket(self, bucket: str) -> bool:
try:
self.tos_client.head_bucket(bucket)
return True
except tos.exceptions.TosClientError as e:
except TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except TosServerError as e:
if e.status_code == TOS_SERVER_RESPONSE_CODE_NOT_FOUND:
return False
else:
Expand Down Expand Up @@ -1594,9 +1576,9 @@ def _exists_object(
try:
self.tos_client.head_object(bucket, key)
return True
except tos.exceptions.TosClientError as e:
except TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except TosServerError as e:
if e.status_code == TOS_SERVER_RESPONSE_CODE_NOT_FOUND:
return False
else:
Expand Down Expand Up @@ -1631,9 +1613,7 @@ def _lsbuckets(self) -> List[dict]:
"""
try:
resp = self.tos_client.list_buckets()
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except (TosClientError, TosServerError) as e:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e
Expand Down Expand Up @@ -1801,9 +1781,7 @@ def _listdir(
all_results.extend(resp.contents + resp.common_prefixes)

return all_results
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except (TosClientError, TosServerError) as e:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e
Expand All @@ -1817,9 +1795,7 @@ def _rm(self, path: str) -> None:

try:
self.tos_client.delete_object(bucket, key)
except tos.exceptions.TosClientError as e:
raise e
except tos.exceptions.TosServerError as e:
except (TosClientError, TosServerError) as e:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e
Expand Down
4 changes: 2 additions & 2 deletions tosfs/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import time
from typing import Any, Generator, Optional, Tuple

import tos
from tos.exceptions import TosServerError

from tosfs.consts import TOS_SERVER_RETRYABLE_ERROR_CODE_SET

Expand Down Expand Up @@ -117,7 +117,7 @@ def retryable_func_wrapper(
for i in range(retries):
try:
return func(*args, **kwargs)
except tos.exceptions.TosServerError as e:
except TosServerError as e:
err = e
from tosfs.core import logger

Expand Down