From 4313c354390fe67fea3207875e8e52c808d4f48f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Koray=20K=C4=B1rl=C4=B1?= Date: Mon, 8 Apr 2019 16:08:47 -0400 Subject: [PATCH 1/3] add embedded --- dcicutils/ff_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dcicutils/ff_utils.py b/dcicutils/ff_utils.py index d41c784bc..35bdb6d39 100644 --- a/dcicutils/ff_utils.py +++ b/dcicutils/ff_utils.py @@ -586,7 +586,9 @@ def expand_es_metadata(uuid_list, key=None, ff_env=None, store_frame='raw', add_ uuid_list (array): Starting node for search, only use uuids. key (dict): standard ff_utils authentication key ff_env (str): standard ff environment string - store_frame ('raw' or 'object'): Depending on use case, can store frame raw or object. + store_frame (str, default 'raw'):Depending on use case, can store frame raw or object or embedded + Note: If you store in embedded, the total collection can have references + to the items that are not in the store add_pc_wfr (bool): Include workflow_runs and linked items (processed/ref files, wf, software...) ignore_field(list): Remove keys from items, so any linking through these fields, ie relations use_generator (bool): Use a generator when getting es. Less memory used but takes longer @@ -648,6 +650,8 @@ def remove_keys(my_dict, remove_list): # get the desired frame from the ES response if store_frame == 'object': frame_resp = remove_keys(es_item['object'], ignore_field) + elif store_frame == 'embedded': + frame_resp = remove_keys(es_item['embedded'], ignore_field) else: frame_resp = remove_keys(es_item['properties'], ignore_field) frame_resp['uuid'] = uuid # uuid is not in properties, so add it From 8be55d880a5f8513a8ccb1822117a36568bbe6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Koray=20K=C4=B1rl=C4=B1?= Date: Mon, 8 Apr 2019 16:17:27 -0400 Subject: [PATCH 2/3] add test expect linked items to be dict --- test/test_ff_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_ff_utils.py b/test/test_ff_utils.py index fdc91a5b7..eb988eac2 100644 --- a/test/test_ff_utils.py +++ b/test/test_ff_utils.py @@ -573,6 +573,17 @@ def test_expand_es_metadata_frame_object(integrated_ff): assert test_item['lab'].startswith('/labs/') +@pytest.mark.integrated +def test_expand_es_metadata_frame_embedded(integrated_ff): + test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f'] + key, ff_env = integrated_ff['ff_key'], integrated_ff['ff_env'] + store, uuids = ff_utils.expand_es_metadata(test_list, store_frame='embedded', key=key, ff_env=ff_env) + # make sure the frame is object (default) + test_item = store['file_processed'][0] + assert isinstance(test_item['lab'], dict) + assert test_item['lab']['@id'].startswith('/labs/') + + @pytest.mark.integrated def test_expand_es_metadata_add_wfrs(integrated_ff): test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f'] From ad1083cff2e3fc567c49d931f6bfe1f93c777981 Mon Sep 17 00:00:00 2001 From: Carl Vitzthum Date: Tue, 9 Apr 2019 09:03:10 -0400 Subject: [PATCH 3/3] Revised test_expand_es_metadata_frame_object_embedded --- test/test_ff_utils.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/test/test_ff_utils.py b/test/test_ff_utils.py index eb988eac2..6b3fd7f45 100644 --- a/test/test_ff_utils.py +++ b/test/test_ff_utils.py @@ -564,24 +564,21 @@ def test_expand_es_metadata(integrated_ff): @pytest.mark.integrated -def test_expand_es_metadata_frame_object(integrated_ff): +def test_expand_es_metadata_frame_object_embedded(integrated_ff): test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f'] key, ff_env = integrated_ff['ff_key'], integrated_ff['ff_env'] - store, uuids = ff_utils.expand_es_metadata(test_list, store_frame='object', key=key, ff_env=ff_env) + store_obj, uuids_obj = ff_utils.expand_es_metadata(test_list, store_frame='object', key=key, ff_env=ff_env) # make sure the frame is object (default) - test_item = store['file_processed'][0] - assert test_item['lab'].startswith('/labs/') - - -@pytest.mark.integrated -def test_expand_es_metadata_frame_embedded(integrated_ff): - test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f'] - key, ff_env = integrated_ff['ff_key'], integrated_ff['ff_env'] - store, uuids = ff_utils.expand_es_metadata(test_list, store_frame='embedded', key=key, ff_env=ff_env) - # make sure the frame is object (default) - test_item = store['file_processed'][0] - assert isinstance(test_item['lab'], dict) - assert test_item['lab']['@id'].startswith('/labs/') + test_item_obj = store_obj['file_processed'][0] + assert test_item_obj['lab'].startswith('/labs/') + + # now test frame=embedded + store_emb, uuids_emb = ff_utils.expand_es_metadata(test_list, store_frame='embedded', key=key, ff_env=ff_env) + test_item_emb = store_emb['file_processed'][0] + assert isinstance(test_item_emb['lab'], dict) + assert test_item_emb['lab']['@id'].startswith('/labs/') + # links found stay the same between embedded and obj + assert set(uuids_obj) == set(uuids_emb) @pytest.mark.integrated