Merge pull request #39 from 4dn-dcic/test_frame_expand_es

test for frame names in expand es metadata

dcicutils/_version.py

@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "0.6.3"
+__version__ = "0.6.4"

dcicutils/ff_utils.py

@@ -575,6 +575,29 @@ def _get_es_metadata(uuids, es_client, filters, sources, chunk_size, key, ff_env
                 yield hit['_source']  # yield individual items from ES
 
 
+def get_schema_names(key=None, ff_env=None):
+    """
+    Create a dictionary of all schema names to item class names
+    i.e. FileFastq: file_fastq
+
+    Args:
+        key (dict):                      standard ff_utils authentication key
+        ff_env (str):                    standard ff environment string
+
+    Returns:
+        dict: contains key schema names and value item class names
+    """
+    auth = get_authentication_with_server(key, ff_env)
+    schema_name = {}
+    profiles = get_metadata('/profiles/', key=auth, add_on='frame=raw')
+    for key, value in profiles.items():
+        # some test schemas in local don't have the id field
+        schema_filename = value.get('id')
+        if schema_filename:
+            schema_name[key] = schema_filename.split('/')[-1][:-5]
+    return schema_name
+
+
 def expand_es_metadata(uuid_list, key=None, ff_env=None, store_frame='raw', add_pc_wfr=False, ignore_field=[],
                        use_generator=False, es_client=None):
     """
@@ -605,6 +628,11 @@ def expand_es_metadata(uuid_list, key=None, ff_env=None, store_frame='raw', add_
     # TODO: if more file types (currently FileFastq and FileProcessed) get workflowrun calculated properties
             we need to add them to the add_from_embedded dictionary.
     """
+    # assert that the used parameter is correct
+    accepted_frames = ['raw', 'object', 'embedded']
+    if store_frame not in accepted_frames:
+        raise ValueError('Invalid frame name "{}", please use one of {}'.format(store_frame, accepted_frames))
+
     # wrap key remover, used multiple times
     def remove_keys(my_dict, remove_list):
         if remove_list:
@@ -619,10 +647,7 @@ def remove_keys(my_dict, remove_list):
         es_client = es_utils.create_es_client(es_url, use_aws_auth=True)
 
     # creates a dictionary of schema names to collection names
-    schema_name = {}
-    profiles = get_metadata('/profiles/', key=auth, add_on='frame=raw')
-    for key, value in profiles.items():
-        schema_name[key] = value['id'].split('/')[-1][:-5]
+    schema_name = get_schema_names(key=auth)
 
     # keep list of fields that only exist in frame embedded (revlinks, calcprops) that you want connected
     if add_pc_wfr:

test/test_ff_utils.py

@@ -548,6 +548,16 @@ def test_get_health_page(integrated_ff):
     assert bad_health_res and 'error' in bad_health_res
 
 
+@pytest.mark.integrated
+def test_get_schema_names(integrated_ff):
+    schema_names = ff_utils.get_schema_names(key=integrated_ff['ff_key'],
+                                             ff_env=integrated_ff['ff_env'])
+    # assert that it gets quite some schemas
+    assert len(schema_names) > 75
+    assert schema_names['FileFastq'] == 'file_fastq'
+    assert schema_names['ExperimentSetReplicate'] == 'experiment_set_replicate'
+
+
 @pytest.mark.integrated
 def test_expand_es_metadata(integrated_ff):
     test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f']
@@ -591,6 +601,15 @@ def test_expand_es_metadata_add_wfrs(integrated_ff):
         assert pos_case in store
 
 
+@pytest.mark.integrated
+def test_expand_es_metadata_complain_wrong_frame(integrated_ff):
+    test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f']
+    key = integrated_ff['ff_key']
+    with pytest.raises(Exception) as exec_info:
+        store, uuids = ff_utils.expand_es_metadata(test_list, add_pc_wfr=True, store_frame='embroiled', key=key)
+    assert str(exec_info.value) == """Invalid frame name "embroiled", please use one of ['raw', 'object', 'embedded']"""
+
+
 @pytest.mark.integrated
 def test_expand_es_metadata_ignore_fields(integrated_ff):
     test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f']