From 42f2d959952dec3c45db8fe3c1657b949d9b5885 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 00:11:25 +0000 Subject: [PATCH] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/check_release_tag.py | 1 + CHANGELOG.md | 50 +++++++++++++------------- disk_objectstore/container.py | 33 +++++++++-------- disk_objectstore/database.py | 1 + disk_objectstore/dataclasses.py | 1 + disk_objectstore/utils.py | 13 ++++--- tests/conftest.py | 1 + tests/test_basic.py | 1 + tests/test_benchmark.py | 1 + tests/test_cli.py | 1 + tests/test_concurrency.py | 1 + tests/test_container.py | 1 + tests/test_examples.py | 1 + tests/test_optional.py | 1 + tests/test_utils.py | 1 + 15 files changed, 59 insertions(+), 49 deletions(-) diff --git a/.github/workflows/check_release_tag.py b/.github/workflows/check_release_tag.py index da36956..62d3db0 100644 --- a/.github/workflows/check_release_tag.py +++ b/.github/workflows/check_release_tag.py @@ -1,4 +1,5 @@ """Check that the GitHub release tag matches the package version.""" + import argparse import pathlib import re diff --git a/CHANGELOG.md b/CHANGELOG.md index 82fde83..5ef7727 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,25 +6,25 @@ This only enforces proper semantic versioning as the last release added a new fu ## v1.1.1 (19 September 2024) -- Added progress bar functionality for repack and pack_all_loose [\[737f9c7\]](https://github.com/aiidateam/disk-objectstore/commit/737f9c71151bf7ac297c6431688b4a75eac91b7c) +- Added progress bar functionality for repack and pack_all_loose [[737f9c7]](https://github.com/aiidateam/disk-objectstore/commit/737f9c71151bf7ac297c6431688b4a75eac91b7c) ## v1.1.0 (7 March 2024) ### Features -- Add functionality to easily create a container backup [\[23c784a\]](https://github.com/aiidateam/disk-objectstore/commit/23c784a221954a1518a3e35affdec53681f809b7) +- Add functionality to easily create a container backup [[23c784a]](https://github.com/aiidateam/disk-objectstore/commit/23c784a221954a1518a3e35affdec53681f809b7) ## v1.0.0 (September 2023) ### Features -- Add support for `whence=2` in `PackedObjectReader.seek` [\[5515ab6\]](https://github.com/aiidateam/disk-objectstore/commit/5515ab6d75581b36ecb3e0b8ff37407e05abefda) -- Add support for changing compression when repacking, and add auto compression heuristics [\[599e87c\]](https://github.com/aiidateam/disk-objectstore/commit/599e87c852427e02062f04f5f3d2276013410710) -- Improve efficiency when accessing packed compressed objects [\[10edd63\]](https://github.com/aiidateam/disk-objectstore/commit/10edd6395455d7c59361e608396b672289d8de58) +- Add support for `whence=2` in `PackedObjectReader.seek` [[5515ab6]](https://github.com/aiidateam/disk-objectstore/commit/5515ab6d75581b36ecb3e0b8ff37407e05abefda) +- Add support for changing compression when repacking, and add auto compression heuristics [[599e87c]](https://github.com/aiidateam/disk-objectstore/commit/599e87c852427e02062f04f5f3d2276013410710) +- Improve efficiency when accessing packed compressed objects [[10edd63]](https://github.com/aiidateam/disk-objectstore/commit/10edd6395455d7c59361e608396b672289d8de58) ### Changes -- A number of API methods changed the return type from bare dictionaries to dataclass instances [\[7a63462\]](https://github.com/aiidateam/disk-objectstore/commit/7a634626ea3e5f35aa3cdd458daf9d8b825d759a) +- A number of API methods changed the return type from bare dictionaries to dataclass instances [[7a63462]](https://github.com/aiidateam/disk-objectstore/commit/7a634626ea3e5f35aa3cdd458daf9d8b825d759a) - `Container.get_object_stream_and_meta -> ObjectMeta` - `Container.get_objects_meta -> ObjectMeta` @@ -35,43 +35,43 @@ This only enforces proper semantic versioning as the last release added a new fu The dataclasses are importable from `disk_objectstore.dataclasses`. -- A number of API methods replaced using `os.path` with `str` paths, for `pathlib.Path` [\[df96142\]](https://github.com/aiidateam/disk-objectstore/commit/df9614236b7d420fb610313d70ffae51e7aead75) +- A number of API methods replaced using `os.path` with `str` paths, for `pathlib.Path` [[df96142]](https://github.com/aiidateam/disk-objectstore/commit/df9614236b7d420fb610313d70ffae51e7aead75) The following methods now return a `pathlib.Path` instance: - `Container.get_folder` - `LazyOpener.path` -- Various improvements to docs and code [\[5ba9316\]](https://github.com/aiidateam/disk-objectstore/commit/5ba93162cd49d9b1ca7149c502349bfb06833255) +- Various improvements to docs and code [[5ba9316]](https://github.com/aiidateam/disk-objectstore/commit/5ba93162cd49d9b1ca7149c502349bfb06833255) ### Devops -- Moving documentation to `sphinx+myst` [\[2002f3c\]](https://github.com/aiidateam/disk-objectstore/commit/2002f3c3ec07f7ff46a04df293c8c9a7dff4db6a) -- Adopt PEP 621 and move build spec to `pyproject.toml` [\[4bd0c4e\]](https://github.com/aiidateam/disk-objectstore/commit/4bd0c4e01eaf3c149d4e11921b7ff4d42a5d5da5) -- Make types more permissive [\[c012056\]](https://github.com/aiidateam/disk-objectstore/commit/c0120568a992b41a55b325f3217d4902b5281070) +- Moving documentation to `sphinx+myst` [[2002f3c]](https://github.com/aiidateam/disk-objectstore/commit/2002f3c3ec07f7ff46a04df293c8c9a7dff4db6a) +- Adopt PEP 621 and move build spec to `pyproject.toml` [[4bd0c4e]](https://github.com/aiidateam/disk-objectstore/commit/4bd0c4e01eaf3c149d4e11921b7ff4d42a5d5da5) +- Make types more permissive [[c012056]](https://github.com/aiidateam/disk-objectstore/commit/c0120568a992b41a55b325f3217d4902b5281070) ### Dependencies -- Add Python 3.11 support [\[afdae26\]](https://github.com/aiidateam/disk-objectstore/commit/afdae261a5849e994b5920ca07665fc6a19f3852) -- Unpin `sqlalchemy` adding support for `>=1.4.22` [\[a2a987f\]](https://github.com/aiidateam/disk-objectstore/commit/a2a987f02a128b7cc265982e102d210e6e17d6f6) -- Removed uneeded `ablog` dependencies [\[8165f58\]](https://github.com/aiidateam/disk-objectstore/commit/8165f58fefdd40b55555eef9a2d40ee280593232) +- Add Python 3.11 support [[afdae26]](https://github.com/aiidateam/disk-objectstore/commit/afdae261a5849e994b5920ca07665fc6a19f3852) +- Unpin `sqlalchemy` adding support for `>=1.4.22` [[a2a987f]](https://github.com/aiidateam/disk-objectstore/commit/a2a987f02a128b7cc265982e102d210e6e17d6f6) +- Removed uneeded `ablog` dependencies [[8165f58]](https://github.com/aiidateam/disk-objectstore/commit/8165f58fefdd40b55555eef9a2d40ee280593232) ## v0.6.0 (September 2021) - ⬆️ UPGRADE: Remove Python support for 3.5 and 3.6, and add support for 3.9. -- ⬆️ UPGRADE: SQLAlchemy v1.4 (with v2 API) [\[#114\]](https://github.com/aiidateam/disk-objectstore/pull/114) -- ✨ NEW: Add basic CLI [\[#117\]](https://github.com/aiidateam/disk-objectstore/pull/117) (see README.md for details) -- 🔧 MAINTAIN: Add type annotations and mypy type checking [\[#113\]](https://github.com/aiidateam/disk-objectstore/pull/113) +- ⬆️ UPGRADE: SQLAlchemy v1.4 (with v2 API) [[#114]](https://github.com/aiidateam/disk-objectstore/pull/114) +- ✨ NEW: Add basic CLI [[#117]](https://github.com/aiidateam/disk-objectstore/pull/117) (see README.md for details) +- 🔧 MAINTAIN: Add type annotations and mypy type checking [[#113]](https://github.com/aiidateam/disk-objectstore/pull/113) ## v0.5.0 (November 2020) -- Various general (but very important) speed improvements [\[#96\]](https://github.com/aiidateam/disk-objectstore/pull/96) [\[#102\]](https://github.com/aiidateam/disk-objectstore/pull/102) -- Add callbacks to a number of functions (e.g. export, add_objects_to_pack, ... to allow showing progress bars or similar indicators [\[#96\]](https://github.com/aiidateam/disk-objectstore/pull/96) -- Implement repacking (at least when not changing hashing or compression) [\[#96\]](https://github.com/aiidateam/disk-objectstore/pull/96) -- Remove `export` function, implement `import_objects` function instead, to be called on the other side (it's more efficient) [\[#96\]](https://github.com/aiidateam/disk-objectstore/pull/96) -- Add support for VACUUMing operations on the SQLite database (very important for efficiency) [\[#96\]](https://github.com/aiidateam/disk-objectstore/pull/96) -- Add support for multiple hashing algorithms [\[#96\]](https://github.com/aiidateam/disk-objectstore/pull/96) -- Add concept of (unique) `container_id` [\[#97\]](https://github.com/aiidateam/disk-objectstore/pull/97) -- Generalize the compression algorithm implementation, and multiple algorithms are supported now [\[#99\]](https://github.com/aiidateam/disk-objectstore/pull/99) +- Various general (but very important) speed improvements [[#96]](https://github.com/aiidateam/disk-objectstore/pull/96) [[#102]](https://github.com/aiidateam/disk-objectstore/pull/102) +- Add callbacks to a number of functions (e.g. export, add_objects_to_pack, ... to allow showing progress bars or similar indicators [[#96]](https://github.com/aiidateam/disk-objectstore/pull/96) +- Implement repacking (at least when not changing hashing or compression) [[#96]](https://github.com/aiidateam/disk-objectstore/pull/96) +- Remove `export` function, implement `import_objects` function instead, to be called on the other side (it's more efficient) [[#96]](https://github.com/aiidateam/disk-objectstore/pull/96) +- Add support for VACUUMing operations on the SQLite database (very important for efficiency) [[#96]](https://github.com/aiidateam/disk-objectstore/pull/96) +- Add support for multiple hashing algorithms [[#96]](https://github.com/aiidateam/disk-objectstore/pull/96) +- Add concept of (unique) `container_id` [[#97]](https://github.com/aiidateam/disk-objectstore/pull/97) +- Generalize the compression algorithm implementation, and multiple algorithms are supported now [[#99]](https://github.com/aiidateam/disk-objectstore/pull/99) ## v0.4.0 (20 July 2020) diff --git a/disk_objectstore/container.py b/disk_objectstore/container.py index dbea118..2f49d97 100644 --- a/disk_objectstore/container.py +++ b/disk_objectstore/container.py @@ -183,14 +183,12 @@ def _get_config_file(self) -> Path: @overload def _get_session( self, create: bool = False, raise_if_missing: Literal[True] = True - ) -> Session: - ... + ) -> Session: ... @overload def _get_session( self, create: bool = False, raise_if_missing: Literal[False] = False - ) -> Session | None: - ... + ) -> Session | None: ... def _get_session( self, create: bool = False, raise_if_missing: bool = False @@ -528,8 +526,7 @@ def _get_objects_stream_meta_generator( hashkeys: Sequence[str], skip_if_missing: bool, with_streams: Literal[False], - ) -> Iterator[tuple[str, ObjectMeta]]: - ... + ) -> Iterator[tuple[str, ObjectMeta]]: ... @overload def _get_objects_stream_meta_generator( @@ -537,8 +534,7 @@ def _get_objects_stream_meta_generator( hashkeys: Sequence[str], skip_if_missing: bool, with_streams: Literal[True], - ) -> Iterator[tuple[str, StreamSeekBytesType | None, ObjectMeta]]: - ... + ) -> Iterator[tuple[str, StreamSeekBytesType | None, ObjectMeta]]: ... def _get_objects_stream_meta_generator( # pylint: disable=too-many-branches,too-many-statements,too-many-locals self, @@ -1323,8 +1319,9 @@ def pack_all_loose( # pylint: disable=too-many-locals,too-many-branches,too-man compress: bool | CompressMode = CompressMode.NO, validate_objects: bool = True, do_fsync: bool = True, - callback: None - | (Callable[[Arg(str, "action"), Arg(Any, "value")], None]) = None, + callback: None | ( + Callable[[Arg(str, "action"), Arg(Any, "value")], None] + ) = None, ) -> None: """Pack all loose objects. @@ -2616,8 +2613,9 @@ def delete_objects(self, hashkeys: list[str]) -> list[str | Any]: def repack( self, compress_mode: CompressMode = CompressMode.KEEP, - callback: None - | (Callable[[Arg(str, "action"), Arg(Any, "value")], None]) = None, + callback: None | ( + Callable[[Arg(str, "action"), Arg(Any, "value")], None] + ) = None, ) -> None: """Perform a repack of all packed objects. @@ -2645,8 +2643,9 @@ def repack_pack( # pylint: disable=too-many-branches,too-many-statements,too-ma self, pack_id: str, compress_mode: CompressMode = CompressMode.KEEP, - callback: None - | (Callable[[Arg(str, "action"), Arg(Any, "value")], None]) = None, + callback: None | ( + Callable[[Arg(str, "action"), Arg(Any, "value")], None] + ) = None, ) -> None: """Perform a repack of a given pack object. @@ -2723,9 +2722,9 @@ def repack_pack( # pylint: disable=too-many-branches,too-many-statements,too-ma source_compressed, ) in session.execute(stmt): # This is the read handle of the bytes in the pack - it might be - read_handle: ( - PackedObjectReader | ZlibStreamDecompresser - ) = PackedObjectReader(read_pack, offset, length) + read_handle: PackedObjectReader | ZlibStreamDecompresser = ( + PackedObjectReader(read_pack, offset, length) + ) # Determine if I should compress or not the destination - this function will # try to do it in a cheap way (e.g. if the source is already compressed, will just diff --git a/disk_objectstore/database.py b/disk_objectstore/database.py index 08c9436..8e1a4c3 100644 --- a/disk_objectstore/database.py +++ b/disk_objectstore/database.py @@ -1,4 +1,5 @@ """Models for the container index file (SQLite DB).""" + from pathlib import Path from typing import Optional diff --git a/disk_objectstore/dataclasses.py b/disk_objectstore/dataclasses.py index 81abb35..1e06c81 100644 --- a/disk_objectstore/dataclasses.py +++ b/disk_objectstore/dataclasses.py @@ -2,6 +2,7 @@ Definition of the dataclasses used as return values of a number of methods. """ + from dataclasses import asdict, dataclass from typing import TYPE_CHECKING, List, Optional, Union diff --git a/disk_objectstore/utils.py b/disk_objectstore/utils.py index 26f7984..6b133d2 100644 --- a/disk_objectstore/utils.py +++ b/disk_objectstore/utils.py @@ -3,6 +3,7 @@ Some might be useful also for end users, like the wrappers to get streams, like the ``LazyOpener``. """ + # pylint: disable= too-many-lines from __future__ import annotations @@ -809,9 +810,9 @@ def __init__( self._decompressor = self.decompressobj_class() self._internal_buffer = b"" self._pos = 0 - self._lazy_uncompressed_stream: None | ( - LazyLooseStream - ) = lazy_uncompressed_stream + self._lazy_uncompressed_stream: None | (LazyLooseStream) = ( + lazy_uncompressed_stream + ) # If True, this class just proxies request to the underlying # uncompressed stream self._use_uncompressed_stream: bool = False @@ -1326,11 +1327,9 @@ def safe_flush_to_disk( fhandle.flush() # Default fsync function, replaced on Mac OS X - _fsync_function: Callable[ - [Any], Any - ] = lambda fileno: os.fsync( # pylint: disable=unnecessary-lambda + _fsync_function: Callable[[Any], Any] = lambda fileno: os.fsync( fileno - ) + ) # pylint: disable=unnecessary-lambda # Flush to disk if hasattr(fcntl, "F_FULLFSYNC") is not None and ( diff --git a/tests/conftest.py b/tests/conftest.py index 38d4e99..5185e24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """Configuration file for pytest tests.""" + import hashlib import os import random diff --git a/tests/test_basic.py b/tests/test_basic.py index 340e16d..10cfd37 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -2,6 +2,7 @@ This is also a way to verify the behavior of the underlying OS/filesystem. """ + import os import subprocess import sys diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index a3fea86..2c6ba9f 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -1,4 +1,5 @@ """Test the performance of the container implementation.""" + import hashlib import random diff --git a/tests/test_cli.py b/tests/test_cli.py index 5b7e2b9..4dfea2f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,5 @@ """Test the CLI commands""" + import platform from pathlib import Path diff --git a/tests/test_concurrency.py b/tests/test_concurrency.py index 3d58f1d..e7ef781 100644 --- a/tests/test_concurrency.py +++ b/tests/test_concurrency.py @@ -1,4 +1,5 @@ """Test of the object-store container module.""" + import os import subprocess import sys diff --git a/tests/test_container.py b/tests/test_container.py index f0c89e5..6596778 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -1,4 +1,5 @@ """Test of the object-store container module.""" + # pylint: disable=too-many-lines,protected-access import dataclasses import functools diff --git a/tests/test_examples.py b/tests/test_examples.py index 65b2e53..6f6d8ea 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,4 +1,5 @@ """Test of the object-store container module.""" + import subprocess import sys import tempfile diff --git a/tests/test_optional.py b/tests/test_optional.py index 3a222b1..b2a2296 100644 --- a/tests/test_optional.py +++ b/tests/test_optional.py @@ -1,5 +1,6 @@ """Additional optional tests to check additional functionality, such as if the library works also with other external optional modules.""" + from pathlib import Path import numpy as np diff --git a/tests/test_utils.py b/tests/test_utils.py index 812748b..ead9716 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ """Test of the utils wrappers.""" + # pylint: disable=too-many-lines,protected-access import functools import hashlib