internal/expand_assetstore_for_testing

src/pump/_bitstream.py

@@ -1,5 +1,5 @@
 import logging
-from ._utils import read_json, time_method, serialize, deserialize, progress_bar, log_before_import, log_after_import
+from ._utils import read_json, time_method, serialize, deserialize, progress_bar, log_before_import, log_after_import, path_exists
 
 _logger = logging.getLogger("pump.bitstream")
 
@@ -49,6 +49,10 @@ def __len__(self):
     def uuid(self, b_id: int):
         return self._id2uuid.get(str(b_id), None)
 
+    @staticmethod
+    def bitstream_path(internal_id: str):
+        return f'{internal_id[:2]}/{internal_id[2:4]}/{internal_id[4:6]}/{internal_id}'
+
     @property
     def imported(self):
         return self._imported['bitstream']
@@ -143,6 +147,12 @@ def _bitstream_import_to(self, env, dspace, metadatas, bitstreamformatregistry,
         log_key = "bitstreams"
         log_before_import(log_key, expected)
 
+        # TODO(jm): fake bitstreams
+        TEST_DEV5 = "http://dev-5.pc" in env["backend"]["endpoint"]
+        if TEST_DEV5 and env["assetstore"] == "":
+            _logger.error(
+                'Location of assetstore folder is not defined but it should be checked!')
+
         for i, b in enumerate(progress_bar(self._bs)):
             b_id = b['bitstream_id']
             b_deleted = b['deleted']
@@ -197,11 +207,13 @@ def _bitstream_import_to(self, env, dspace, metadatas, bitstreamformatregistry,
             }
 
             # TODO(jm): fake bitstreams
-            TEST_DEV5 = "http://dev-5.pc" in env["backend"]["endpoint"]
-            if TEST_DEV5:
+            path = self.bitstream_path(params['internal_id'])
+            if TEST_DEV5 and not path_exists(f'{env["assetstore"]}{path}'):
                 data['sizeBytes'] = 1748
                 data['checkSum'] = {
-                    'checkSumAlgorithm': b['checksum_algorithm'], 'value': '8a4605be74aa9ea9d79846c1fba20a33'}
+                    'checkSumAlgorithm': b['checksum_algorithm'],
+                    'value': '8a4605be74aa9ea9d79846c1fba20a33'
+                }
                 params['internal_id'] = '77893754617268908529226218097860272513'
 
             # if bitstream has bundle, set bundle_id from None to id

src/pump/_utils.py

@@ -3,6 +3,8 @@
 import logging
 from datetime import datetime, timezone
 from time import time as time_fnc
+from pathlib import Path
+
 _logger = logging.getLogger("pump.utils")
 
 
@@ -126,3 +128,8 @@ def log_before_import(msg: str, expected: int):
 def log_after_import(msg: str, expected: int, imported: int):
     prefix = "OK " if expected == imported else "!!! WARN !!! "
     _logger.info(f"{prefix}Imported [{imported: >4d}] {msg}")
+
+
+def path_exists(path):
+    path_obj = Path(path)
+    return path_obj.exists()

src/repo_import.py

@@ -52,6 +52,9 @@ def deserialize(resume: bool, obj, cache_file: str) -> bool:
     parser.add_argument('--config',
                         help='Update configs',
                         required=False, type=str, action='append')
+    parser.add_argument('--assetstore',
+                        help='Location of assetstore folder',
+                        required=False, type=str, default="")
 
     args = parser.parse_args()
     s = time.time()
@@ -67,6 +70,9 @@ def deserialize(resume: bool, obj, cache_file: str) -> bool:
             new_val = type(prev_val)(v)
         set_key(k, new_val, env)
 
+    # add assetstore folder location to env
+    env["assetstore"] = args.assetstore
+
     # just in case
     # verify_disabled_mailserver()