diff --git a/src/pump/_bitstream.py b/src/pump/_bitstream.py index acb4c40..d564177 100644 --- a/src/pump/_bitstream.py +++ b/src/pump/_bitstream.py @@ -1,5 +1,5 @@ import logging -from ._utils import read_json, time_method, serialize, deserialize, progress_bar, log_before_import, log_after_import +from ._utils import read_json, time_method, serialize, deserialize, progress_bar, log_before_import, log_after_import, path_exists _logger = logging.getLogger("pump.bitstream") @@ -49,6 +49,10 @@ def __len__(self): def uuid(self, b_id: int): return self._id2uuid.get(str(b_id), None) + @staticmethod + def bitstream_path(internal_id: str): + return f'{internal_id[:2]}/{internal_id[2:4]}/{internal_id[4:6]}/{internal_id}' + @property def imported(self): return self._imported['bitstream'] @@ -143,6 +147,12 @@ def _bitstream_import_to(self, env, dspace, metadatas, bitstreamformatregistry, log_key = "bitstreams" log_before_import(log_key, expected) + # TODO(jm): fake bitstreams + TEST_DEV5 = "http://dev-5.pc" in env["backend"]["endpoint"] + if TEST_DEV5 and env["assetstore"] == "": + _logger.error( + 'Location of assetstore folder is not defined but it should be checked!') + for i, b in enumerate(progress_bar(self._bs)): b_id = b['bitstream_id'] b_deleted = b['deleted'] @@ -197,11 +207,13 @@ def _bitstream_import_to(self, env, dspace, metadatas, bitstreamformatregistry, } # TODO(jm): fake bitstreams - TEST_DEV5 = "http://dev-5.pc" in env["backend"]["endpoint"] - if TEST_DEV5: + path = self.bitstream_path(params['internal_id']) + if TEST_DEV5 and not path_exists(f'{env["assetstore"]}{path}'): data['sizeBytes'] = 1748 data['checkSum'] = { - 'checkSumAlgorithm': b['checksum_algorithm'], 'value': '8a4605be74aa9ea9d79846c1fba20a33'} + 'checkSumAlgorithm': b['checksum_algorithm'], + 'value': '8a4605be74aa9ea9d79846c1fba20a33' + } params['internal_id'] = '77893754617268908529226218097860272513' # if bitstream has bundle, set bundle_id from None to id diff --git a/src/pump/_utils.py b/src/pump/_utils.py index 3709d1c..62b8998 100644 --- a/src/pump/_utils.py +++ b/src/pump/_utils.py @@ -3,6 +3,8 @@ import logging from datetime import datetime, timezone from time import time as time_fnc +from pathlib import Path + _logger = logging.getLogger("pump.utils") @@ -126,3 +128,8 @@ def log_before_import(msg: str, expected: int): def log_after_import(msg: str, expected: int, imported: int): prefix = "OK " if expected == imported else "!!! WARN !!! " _logger.info(f"{prefix}Imported [{imported: >4d}] {msg}") + + +def path_exists(path): + path_obj = Path(path) + return path_obj.exists() diff --git a/src/repo_import.py b/src/repo_import.py index 9f7339f..34a0d6d 100644 --- a/src/repo_import.py +++ b/src/repo_import.py @@ -52,6 +52,9 @@ def deserialize(resume: bool, obj, cache_file: str) -> bool: parser.add_argument('--config', help='Update configs', required=False, type=str, action='append') + parser.add_argument('--assetstore', + help='Location of assetstore folder', + required=False, type=str, default="") args = parser.parse_args() s = time.time() @@ -67,6 +70,9 @@ def deserialize(resume: bool, obj, cache_file: str) -> bool: new_val = type(prev_val)(v) set_key(k, new_val, env) + # add assetstore folder location to env + env["assetstore"] = args.assetstore + # just in case # verify_disabled_mailserver()