diff --git a/dcicutils/_version.py b/dcicutils/_version.py index 43850248a..871f68450 100644 --- a/dcicutils/_version.py +++ b/dcicutils/_version.py @@ -1,4 +1,4 @@ """Version information.""" # The following line *must* be the last in the module, exactly as formatted: -__version__ = "0.6.3" +__version__ = "0.6.4" diff --git a/dcicutils/ff_utils.py b/dcicutils/ff_utils.py index 35bdb6d39..234ced7f3 100644 --- a/dcicutils/ff_utils.py +++ b/dcicutils/ff_utils.py @@ -575,6 +575,29 @@ def _get_es_metadata(uuids, es_client, filters, sources, chunk_size, key, ff_env yield hit['_source'] # yield individual items from ES +def get_schema_names(key=None, ff_env=None): + """ + Create a dictionary of all schema names to item class names + i.e. FileFastq: file_fastq + + Args: + key (dict): standard ff_utils authentication key + ff_env (str): standard ff environment string + + Returns: + dict: contains key schema names and value item class names + """ + auth = get_authentication_with_server(key, ff_env) + schema_name = {} + profiles = get_metadata('/profiles/', key=auth, add_on='frame=raw') + for key, value in profiles.items(): + # some test schemas in local don't have the id field + schema_filename = value.get('id') + if schema_filename: + schema_name[key] = schema_filename.split('/')[-1][:-5] + return schema_name + + def expand_es_metadata(uuid_list, key=None, ff_env=None, store_frame='raw', add_pc_wfr=False, ignore_field=[], use_generator=False, es_client=None): """ @@ -605,6 +628,11 @@ def expand_es_metadata(uuid_list, key=None, ff_env=None, store_frame='raw', add_ # TODO: if more file types (currently FileFastq and FileProcessed) get workflowrun calculated properties we need to add them to the add_from_embedded dictionary. """ + # assert that the used parameter is correct + accepted_frames = ['raw', 'object', 'embedded'] + if store_frame not in accepted_frames: + raise ValueError('Invalid frame name "{}", please use one of {}'.format(store_frame, accepted_frames)) + # wrap key remover, used multiple times def remove_keys(my_dict, remove_list): if remove_list: @@ -619,10 +647,7 @@ def remove_keys(my_dict, remove_list): es_client = es_utils.create_es_client(es_url, use_aws_auth=True) # creates a dictionary of schema names to collection names - schema_name = {} - profiles = get_metadata('/profiles/', key=auth, add_on='frame=raw') - for key, value in profiles.items(): - schema_name[key] = value['id'].split('/')[-1][:-5] + schema_name = get_schema_names(key=auth) # keep list of fields that only exist in frame embedded (revlinks, calcprops) that you want connected if add_pc_wfr: diff --git a/test/test_ff_utils.py b/test/test_ff_utils.py index 6b3fd7f45..b3bfeb5b9 100644 --- a/test/test_ff_utils.py +++ b/test/test_ff_utils.py @@ -548,6 +548,16 @@ def test_get_health_page(integrated_ff): assert bad_health_res and 'error' in bad_health_res +@pytest.mark.integrated +def test_get_schema_names(integrated_ff): + schema_names = ff_utils.get_schema_names(key=integrated_ff['ff_key'], + ff_env=integrated_ff['ff_env']) + # assert that it gets quite some schemas + assert len(schema_names) > 75 + assert schema_names['FileFastq'] == 'file_fastq' + assert schema_names['ExperimentSetReplicate'] == 'experiment_set_replicate' + + @pytest.mark.integrated def test_expand_es_metadata(integrated_ff): test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f'] @@ -591,6 +601,15 @@ def test_expand_es_metadata_add_wfrs(integrated_ff): assert pos_case in store +@pytest.mark.integrated +def test_expand_es_metadata_complain_wrong_frame(integrated_ff): + test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f'] + key = integrated_ff['ff_key'] + with pytest.raises(Exception) as exec_info: + store, uuids = ff_utils.expand_es_metadata(test_list, add_pc_wfr=True, store_frame='embroiled', key=key) + assert str(exec_info.value) == """Invalid frame name "embroiled", please use one of ['raw', 'object', 'embedded']""" + + @pytest.mark.integrated def test_expand_es_metadata_ignore_fields(integrated_ff): test_list = ['7f9eb396-5c1a-4c5e-aebf-28ea39d6a50f']