Skip to content

Commit

Permalink
make refiners.conversion.utils.Hub.expected_sha256 optional
Browse files Browse the repository at this point in the history
  • Loading branch information
Laurent2916 committed Nov 6, 2024
1 parent d90bb25 commit dc08937
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions src/refiners/conversion/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ def __init__(
self,
repo_id: str,
filename: str,
expected_sha256: str,
revision: str = "main",
expected_sha256: str | None = None,
download_url: str | None = None,
) -> None:
"""Initialize the HubPath.
Expand All @@ -79,7 +79,7 @@ def __init__(
self.repo_id = repo_id
self.filename = filename
self.revision = revision
self.expected_sha256 = expected_sha256.lower()
self.expected_sha256 = expected_sha256.lower() if expected_sha256 is not None else None
self.override_download_url = download_url

@staticmethod
Expand Down Expand Up @@ -135,6 +135,7 @@ def local_hash(self) -> str:

def check_local_hash(self) -> bool:
"""Check if the sha256 hash of the file in the local hub is correct."""
assert self.expected_sha256 is not None, f"{self.repo_id}/{self.filename} has no expected sha256 hash"
if self.expected_sha256 != self.local_hash:
logging.warning(f"{self.local_path} local sha256 mismatch, {self.local_hash} != {self.expected_sha256}")
return False
Expand All @@ -144,6 +145,7 @@ def check_local_hash(self) -> bool:

def check_remote_hash(self) -> bool:
"""Check if the sha256 hash of the file on the hf hub is correct."""
assert self.expected_sha256 is not None, f"{self.repo_id}/{self.filename} has no expected sha256 hash"
if self.expected_sha256 != self.hf_sha256_hash:
logging.warning(
f"{self.local_path} remote sha256 mismatch, {self.hf_sha256_hash} != {self.expected_sha256}"
Expand Down

0 comments on commit dc08937

Please sign in to comment.