Merge pull request #38 from 4dn-dcic/expand_with_embedded

Merge branch 'master' into expand_with_embedded

dcicutils/ff_utils.py

@@ -586,7 +586,9 @@ def expand_es_metadata(uuid_list, key=None, ff_env=None, store_frame='raw', add_
         uuid_list (array):               Starting node for search, only use uuids.
         key (dict):                      standard ff_utils authentication key
         ff_env (str):                    standard ff environment string
-        store_frame ('raw' or 'object'): Depending on use case, can store frame raw or object.
+        store_frame (str, default 'raw'):Depending on use case, can store frame raw or object or embedded
+                                         Note: If you store in embedded, the total collection can have references
+                                         to the items that are not in the store
         add_pc_wfr (bool):               Include workflow_runs and linked items (processed/ref files, wf, software...)
         ignore_field(list):              Remove keys from items, so any linking through these fields, ie relations
         use_generator (bool):            Use a generator when getting es. Less memory used but takes longer
@@ -648,6 +650,8 @@ def remove_keys(my_dict, remove_list):
                 # get the desired frame from the ES response
                 if store_frame == 'object':
                     frame_resp = remove_keys(es_item['object'], ignore_field)
+                elif store_frame == 'embedded':
+                    frame_resp = remove_keys(es_item['embedded'], ignore_field)
                 else:
                     frame_resp = remove_keys(es_item['properties'], ignore_field)
                     frame_resp['uuid'] = uuid  # uuid is not in properties, so add it

test/test_ff_utils.py

@@ -564,13 +564,21 @@ def test_expand_es_metadata(integrated_ff):
 
 
 @pytest.mark.integrated
-def test_expand_es_metadata_frame_object(integrated_ff):
+def test_expand_es_metadata_frame_object_embedded(integrated_ff):
     test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f']
     key, ff_env = integrated_ff['ff_key'], integrated_ff['ff_env']
-    store, uuids = ff_utils.expand_es_metadata(test_list, store_frame='object', key=key, ff_env=ff_env)
+    store_obj, uuids_obj = ff_utils.expand_es_metadata(test_list, store_frame='object', key=key, ff_env=ff_env)
     # make sure the frame is object (default)
-    test_item = store['file_processed'][0]
-    assert test_item['lab'].startswith('/labs/')
+    test_item_obj = store_obj['file_processed'][0]
+    assert test_item_obj['lab'].startswith('/labs/')
+
+    # now test frame=embedded
+    store_emb, uuids_emb = ff_utils.expand_es_metadata(test_list, store_frame='embedded', key=key, ff_env=ff_env)
+    test_item_emb = store_emb['file_processed'][0]
+    assert isinstance(test_item_emb['lab'], dict)
+    assert test_item_emb['lab']['@id'].startswith('/labs/')
+    # links found stay the same between embedded and obj
+    assert set(uuids_obj) == set(uuids_emb)
 
 
 @pytest.mark.integrated