From a4c55ce51c2cd0b7475b8ee83d09f870b05d3488 Mon Sep 17 00:00:00 2001 From: Ali Date: Tue, 17 Sep 2024 16:05:20 +0200 Subject: [PATCH] review applied --- disk_objectstore/container.py | 27 +++++++++++++++++ tests/test_container.py | 57 +++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/disk_objectstore/container.py b/disk_objectstore/container.py index 05f5481..0f4393c 100644 --- a/disk_objectstore/container.py +++ b/disk_objectstore/container.py @@ -1340,6 +1340,15 @@ def pack_all_loose( # pylint: disable=too-many-locals,too-many-branches,too-man Needed to guarantee that data will be there even in the case of a power loss. Set to False if you don't need such a guarantee (anyway the loose version will be kept, so often this guarantee is not strictly needed). + :param callback: a callback function that can be used to report progress. + The callback function should accept two arguments: a string with the action being performed + and the value of the action. The action can be "init" (initialization), + "update" (update of the progress), or "close" (finalization). + In case of "init", the value is a dictionary with the key "total" as the total size of the operation + and key "description" as the label of the operation. + In case of "update", the value is amount of the operation that has been completed. + In case of "close", the value is None. + return value of the callback function is ignored. """ hash_type = self.hash_type if validate_objects else None @@ -2611,6 +2620,15 @@ def repack( This is a maintenance operation. :param compress_mode: see docstring of ``repack_pack``. + :param callback: a callback function that can be used to report progress. + The callback function should accept two arguments: a string with the action being performed + and the value of the action. The action can be "init" (initialization), + "update" (update of the progress), or "close" (finalization). + In case of "init", the value is a dictionary with the key "total" as the total size of the operation + and key "description" as the label of the operation. + In case of "update", the value is amount of the operation that has been completed. + In case of "close", the value is None. + return value of the callback function is ignored. """ for pack_id in self._list_packs(): self.repack_pack(pack_id, compress_mode=compress_mode, callback=callback) @@ -2631,6 +2649,15 @@ def repack_pack( # pylint: disable=too-many-branches,too-many-statements,too-ma preserves the same compression (this means that repacking is *much* faster as it can simply transfer the bytes without decompressing everything first, and recompressing it back again). + :param callback: a callback function that can be used to report progress. + The callback function should accept two arguments: a string with the action being performed + and the value of the action. The action can be "init" (initialization), + "update" (update of the progress), or "close" (finalization). + In case of "init", the value is a dictionary with the key "total" as the total size of the operation + and key "description" as the label of the operation. + In case of "update", the value is amount of the operation that has been completed. + In case of "close", the value is None. + return value of the callback function is ignored. """ assert ( pack_id != self._REPACK_PACK_ID diff --git a/tests/test_container.py b/tests/test_container.py index dae8841..f0c89e5 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -3446,6 +3446,36 @@ def test_repack_auto_many_sizes(temp_container: Container, start_compressed): assert issues.is_valid() +def test_repack_progress_bar(temp_container: Container): + """Check that the progress bar is correctly updated when packing all loose objects.""" + objects = [] + for length in range(0, 100): + objects.append(b"a" * length) + + temp_container.add_objects_to_pack(objects, compress=False) + + passed_updates = 0 + passed_total = 0 + + def progress(action, value): + """A dummy progress function.""" + if action == "init": + assert ( + value["total"] == temp_container.get_total_size()["total_size_packed"] + ) + nonlocal passed_total + passed_total = value["total"] + assert value["description"] == "Repack 0" + elif action == "update": + isinstance(value, (int, float)) + nonlocal passed_updates + passed_updates += value + elif action == "close": + assert passed_updates == passed_total + + temp_container.repack(callback=progress) + + @pytest.mark.parametrize("compress_mode", [True, False] + list(CompressMode)) def test_pack_all_loose_compress_modes(temp_container: Container, compress_mode): """Check that pack_all_loose() uses the correct compression mode. @@ -3518,6 +3548,33 @@ def test_pack_all_loose_many(temp_container): assert retrieved == expected +def test_pack_all_loose_progress_bar(temp_container): + """Check that the progress bar is correctly updated when packing all loose objects.""" + # Add 10 objects + for idx in range(10): + content = f"{idx}".encode() + temp_container.add_object(content) + + passed_updates = 0 + passed_total = 0 + + def progress(action, value): + """A dummy progress function.""" + if action == "init": + assert value["total"] == temp_container.get_total_size()["total_size_loose"] + nonlocal passed_total + passed_total = value["total"] + assert value["description"] == "Packing loose objects" + elif action == "update": + isinstance(value, (int, float)) + nonlocal passed_updates + passed_updates += value + elif action == "close": + assert passed_updates == passed_total + + temp_container.pack_all_loose(callback=progress) + + def test_container_id(temp_container): """Check the creation of unique container IDs.""" old_container_id = temp_container.container_id