diff --git a/src/lib/datacite_doi_helper.py b/src/lib/datacite_doi_helper.py index 2e82a6d1..f57f9d91 100644 --- a/src/lib/datacite_doi_helper.py +++ b/src/lib/datacite_doi_helper.py @@ -7,7 +7,7 @@ import logging from flask import Flask from api.datacite_api import DataCiteApi -from hubmap_sdk import EntitySdk +from hubmap_sdk import Entity, EntitySdk from hubmap_commons.exceptions import HTTPException import ast @@ -120,7 +120,6 @@ def build_doi_creators(self, dataset: object) -> list: return creators - """ Register a draft DOI with DataCite @@ -181,21 +180,28 @@ def create_dataset_draft_doi(self, dataset: dict, check_publication_status=True) raise KeyError('Either the entity_type of the given Dataset is missing or the entity is not a Dataset') """ - Move the DOI state from draft to findable, meaning publish this dataset - + Move the DOI state from draft to findable (publish) in DataCite + Parameters ---------- dataset: dict The dataset dict to be published user_token: str The user's globus nexus token - + Returns ------- dict - The published datset entity dict with updated DOI properties + The updated DOI properties + + Raises + ------ + requests.exceptions.RequestException + If the request to entity-api fails + KeyError + If the entity_type of the given Dataset is missing or the entity is not a Dataset """ - def move_doi_state_from_draft_to_findable(self, dataset: dict, user_token: str) -> object: + def move_doi_state_from_draft_to_findable(self, dataset: dict, user_token: str) -> dict: if ('entity_type' in dataset) and (dataset['entity_type'] == 'Dataset'): datacite_api = DataCiteApi(self.datacite_repository_id, self.datacite_repository_password, self.datacite_hubmap_prefix, self.datacite_api_url, self.entity_api_url) @@ -207,19 +213,16 @@ def move_doi_state_from_draft_to_findable(self, dataset: dict, user_token: str) logger.debug("======resulting json from DataCite======") logger.debug(doi_data) - # Then update the dataset DOI properties via entity-api after the DOI gets published - try: - doi_name = datacite_api.build_doi_name(dataset['sennet_id']) - entity_api = EntitySdk(user_token, self.entity_api_url) - updated_dataset = self.update_dataset_after_doi_published(dataset['uuid'], doi_name, entity_api) - - return updated_dataset - except requests.exceptions.RequestException as e: - raise requests.exceptions.RequestException(e) + doi_name = datacite_api.build_doi_name(dataset['sennet_id']) + doi_info = { + 'registered_doi': doi_name, + 'doi_url': f'https://doi.org/{doi_name}' + } + return doi_info else: # Log the full stack trace, prepend a line with our message logger.exception(f"Unable to publish DOI for dataset {dataset['uuid']} via DataCite") - logger.debug(f'======Status code from DataCite {response.status_code} ======') + logger.debug(f'======Status code from DataCite {response.status_code}======') logger.debug("======response text from DataCite======") logger.debug(response.text) @@ -229,8 +232,8 @@ def move_doi_state_from_draft_to_findable(self, dataset: dict, user_token: str) raise KeyError('Either the entity_type of the given Dataset is missing or the entity is not a Dataset') """ - Update the dataset's properties after DOI is published (Draft -> Findable) - + Update the dataset's properties in Entity-API after DOI is published (Draft -> Findable) + Parameters ---------- dataset_uuid: str @@ -239,35 +242,32 @@ def move_doi_state_from_draft_to_findable(self, dataset: dict, user_token: str) The registered doi: prefix/suffix entity_api The EntitySdk object instance - + Returns ------- - dict - The entity dict with updated DOI properties - """ - def update_dataset_after_doi_published(self, dataset_uuid: str, doi_name: str, entity_api: EntitySdk) -> object: + hubmap_sdk.Entity + The updated dataset entity + Raises + ------ + requests.exceptions.RequestException + If the request to entity-api fails + """ + def update_dataset_after_doi_published(self, dataset_uuid: dict, doi_info: str, entity_api: EntitySdk) -> Entity: # Update the registered_doi, and doi_url properties after DOI made findable # Changing Dataset.status to "Published" and setting the published_* properties # are handled by another script # See https://github.com/hubmapconsortium/ingest-ui/issues/354 - dataset_properties_to_update = { - 'registered_doi': doi_name, - 'doi_url': f'https://doi.org/{doi_name}' - } - try: - entity = entity_api.update_entity(dataset_uuid, dataset_properties_to_update) + # Entity update via PUT call only returns a json message, no entity details + result = entity_api.update_entity(dataset_uuid, doi_info) logger.info("======The dataset {dataset['uuid']} has been updated with DOI info======") - updated_entity = vars(entity) - logger.debug("======updated_entity======") - logger.debug(updated_entity) - return updated_entity - + logger.info(doi_info) + return result except HTTPException as e: # Log the full stack trace, prepend a line with our message logger.exception(f"Unable to update the DOI properties of dataset {dataset_uuid}") - logger.debug(f'======Status code from DataCite {e.status_code} ======') + logger.debug(f'======Status code from DataCite {e.status_code}======') logger.debug("======response text from entity-api======") logger.debug(e.description) @@ -283,7 +283,7 @@ def update_dataset_after_doi_published(self, dataset_uuid: str, doi_name: str, e try: user_token = sys.argv[1] - except IndexError as e: + except IndexError: msg = "Missing user token argument" # Log the full stack trace, prepend a line with our message logger.exception(msg) @@ -295,7 +295,7 @@ def update_dataset_after_doi_published(self, dataset_uuid: str, doi_name: str, e count = 1 for dataset_uuid in datasets: - logger.debug(f"Begin {count}: ========================= {dataset_uuid} =========================") + logger.debug(f"Begin {count}: ========================={dataset_uuid}=========================") try: entity = entity_api.get_entity_by_id(dataset_uuid) dataset = vars(entity) @@ -306,7 +306,7 @@ def update_dataset_after_doi_published(self, dataset_uuid: str, doi_name: str, e logger.debug("Create Draft DOI") # DISABLED - #data_cite_doi_helper.create_dataset_draft_doi(dataset) + # data_cite_doi_helper.create_dataset_draft_doi(dataset) except Exception as e: logger.exception(e) sys.exit(e) @@ -316,7 +316,7 @@ def update_dataset_after_doi_published(self, dataset_uuid: str, doi_name: str, e # DISABLED # To publish an existing draft DOI (change the state from draft to findable) - #data_cite_doi_helper.move_doi_state_from_draft_to_findable(dataset, user_token) + # data_cite_doi_helper.move_doi_state_from_draft_to_findable(dataset, user_token) except Exception as e: logger.exception(e) sys.exit(e) @@ -330,8 +330,8 @@ def update_dataset_after_doi_published(self, dataset_uuid: str, doi_name: str, e logger.debug("======response text from entity-api======") logger.debug(e.description) - logger.debug(f"End {count}: ========================= {dataset_uuid} =========================") + logger.debug(f"End {count}: ========================={dataset_uuid}=========================") time.sleep(1) - count = count + 1 \ No newline at end of file + count = count + 1 diff --git a/src/lib/services.py b/src/lib/services.py index 45a2b589..e1b790ec 100644 --- a/src/lib/services.py +++ b/src/lib/services.py @@ -1,5 +1,6 @@ -from typing import Optional, Union +from typing import List, Optional, Union +import requests from flask import current_app from hubmap_sdk import Entity, EntitySdk, SearchSdk from hubmap_sdk.sdk_helper import HTTPException as SDKException @@ -90,3 +91,44 @@ def get_entity_from_search_api( except SDKException: raise + + +def get_associated_sources_from_dataset( + dataset_id: str, token: str, as_dict: bool = False +) -> Union[List[Entity], dict]: + """Get the associated sources for the given dataset. + + Parameters + ---------- + dataset_id : str + The uuid of the dataset. + token : str + The groups token for the request. + as_dict : bool, optional + Should entity be returned as a dictionary, by default False + + Returns + ------- + Union[List[Entity], dict] + The associated sources for the given dataset. + + Raises + ------ + hubmap_sdk.sdk_helper.HTTPException + If the entiti-api request fails or entity not found + """ + entity_api_url = current_app.config["ENTITY_WEBSERVICE_URL"] + url = f"{entity_api_url}/datasets/{dataset_id}/sources" + headers = {"Authorization": f"Bearer {token}"} + res = requests.get(url, headers=headers) + if not res.ok: + raise SDKException(f"Failed to get associated source for dataset {dataset_id}") + body = res.json() + + if as_dict: + return body + + if isinstance(body, list): + return [Entity(entity) for entity in res.json()] + + return [Entity(body)] diff --git a/src/requirements.txt b/src/requirements.txt index ef4a0c33..aa123b1b 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,6 +1,6 @@ Flask==2.1.3 globus_sdk==2.0.1 -hubmap-sdk==1.0.4 +hubmap-sdk==1.0.7 Werkzeug==2.3.7 # For interacting with memcached diff --git a/src/routes/entity_CRUD/__init__.py b/src/routes/entity_CRUD/__init__.py index 35b22a76..ea095587 100644 --- a/src/routes/entity_CRUD/__init__.py +++ b/src/routes/entity_CRUD/__init__.py @@ -1,14 +1,11 @@ -from flask import Blueprint, jsonify, request, Response, current_app, abort, json +from flask import Blueprint, jsonify, request, Response, current_app, json import logging import requests import os import re import datetime import time -from typing import List -import urllib.request -import yaml -from hubmap_sdk import EntitySdk +from hubmap_sdk import Entity, EntitySdk from werkzeug import utils from operator import itemgetter from threading import Thread @@ -18,8 +15,9 @@ from hubmap_commons import file_helper as commons_file_helper from hubmap_commons import string_helper from atlas_consortia_commons.rest import * +from atlas_consortia_commons.rest import abort_bad_req, abort_forbidden, abort_not_found from atlas_consortia_commons.string import equals -from atlas_consortia_commons.object import includes, enum_val_lower +from atlas_consortia_commons.object import enum_val_lower from lib.exceptions import ResponseException from lib.file_upload_helper import UploadFileHelper @@ -39,7 +37,7 @@ from lib.ontology import Ontology, get_dataset_types_ep, get_organ_types_ep from lib.file import get_csv_records, get_base_path, check_upload, ln_err, files_exist - +from lib.services import get_associated_sources_from_dataset @entity_CRUD_blueprint.route('/datasets', methods=['POST']) @@ -725,26 +723,26 @@ def dataset_data_status(): def publish_datastage(identifier): try: auth_helper = AuthHelper.instance() - #dataset_helper = DatasetHelper(current_app.config) - user_info = auth_helper.getUserInfoUsingRequest(request, getGroups = True) + user_info = auth_helper.getUserInfoUsingRequest(request, getGroups=True) if user_info is None: return Response("Unable to obtain user information for auth token", 401) if isinstance(user_info, Response): return user_info if 'hmgroupids' not in user_info: - return Response("User has no valid group information to authorize publication.", 403) + abort_forbidden('User has no valid group information to authorize publication.') if not auth_helper.has_data_admin_privs(auth_helper.getUserTokenFromRequest(request, getGroups=True)): - return Response("User must be a member of the SenNet Data Admin group to publish data.", 403) + abort_forbidden('User must be a member of the SenNet Data Admin group to publish data.') if identifier is None or len(identifier) == 0: abort_bad_req('identifier parameter is required to publish a dataset') - r = requests.get(current_app.config['UUID_WEBSERVICE_URL'] + "uuid/" + identifier, - headers={'Authorization': request.headers["AUTHORIZATION"]}) + url = commons_file_helper.ensureTrailingSlashURL(current_app.config['UUID_WEBSERVICE_URL']) + "uuid/" + identifier + r = requests.get(url, headers={'Authorization': request.headers["AUTHORIZATION"]}) if r.ok is False: - raise ValueError("Cannot find specimen with identifier: " + identifier) + abort_not_found("Cannot find specimen with identifier: " + identifier) + dataset_uuid = json.loads(r.text)['hm_uuid'] is_primary = dataset_is_primary(dataset_uuid) suspend_indexing_and_acls = string_helper.isYes(request.args.get('suspend-indexing-and-acls')) @@ -754,16 +752,9 @@ def publish_datastage(identifier): sources_to_reindex = [] with Neo4jHelper.get_instance().session() as neo_session: - #recds = session.run("Match () Return 1 Limit 1") - #for recd in recds: - # if recd[0] == 1: - # is_connected = True - # else: - # is_connected = False - - #look at all of the ancestors - #gather uuids of ancestors that need to be switched to public access_level - #grab the id of the source ancestor to use for reindexing + # look at all of the ancestors + # gather uuids of ancestors that need to be switched to public access_level + # grab the id of the source ancestor to use for reindexing q = f"MATCH (dataset:Dataset {{uuid: '{dataset_uuid}'}})-[:WAS_GENERATED_BY]->(e1)-[:USED|WAS_GENERATED_BY*]->(all_ancestors:Entity) RETURN distinct all_ancestors.uuid as uuid, all_ancestors.entity_type as entity_type, all_ancestors.dataset_type as dataset_type, all_ancestors.data_access_level as data_access_level, all_ancestors.status as status, all_ancestors.metadata as metadata" rval = neo_session.run(q).data() uuids_for_public = [] @@ -799,13 +790,21 @@ def publish_datastage(identifier): uuids_for_public.append(uuid) elif entity_type == 'Dataset': if status != 'Published': - return Response(f"{dataset_uuid} has an ancestor dataset that has not been Published. Will not Publish. Ancestor dataset is: {uuid}", 400) + abort_bad_req(f"{dataset_uuid} has an ancestor dataset that has not been Published. Will not Publish. Ancestor dataset is: {uuid}") if has_source is False: - return Response(f"{dataset_uuid}: no source found for dataset, will not Publish") + abort_bad_req(f"{dataset_uuid}: no source found for dataset, will not Publish") + + # get info for the dataset to be published + q = ( + f"MATCH (e:Dataset {{uuid: '{dataset_uuid}'}}) RETURN " + "e.uuid as uuid, e.entity_type as entitytype, e.status as status, " + "e.data_access_level as data_access_level, e.group_uuid as group_uuid, " + "e.contacts as contacts, e.contributors as contributors" + ) + if is_primary: + q += ", e.ingest_metadata as ingest_metadata" - #get info for the dataset to be published - q = f"MATCH (e:Dataset {{uuid: '{dataset_uuid}'}}) RETURN e.uuid as uuid, e.entity_type as entitytype, e.status as status, e.data_access_level as data_access_level, e.group_uuid as group_uuid, e.contacts as contacts, e.contributors as contributors" rval = neo_session.run(q).data() dataset_status = rval[0]['status'] dataset_entitytype = rval[0]['entitytype'] @@ -813,15 +812,22 @@ def publish_datastage(identifier): dataset_group_uuid = rval[0]['group_uuid'] dataset_contacts = rval[0]['contacts'] dataset_contributors = rval[0]['contributors'] + dataset_ingest_metadata_dict = None + if is_primary: + dataset_ingest_metadata = rval[0].get('ingest_metadata') + if dataset_ingest_metadata is not None: + dataset_ingest_metadata_dict: dict = string_helper.convert_str_literal(dataset_ingest_metadata) + logger.info(f"publish_datastage; ingest_metadata: {dataset_ingest_metadata_dict}") + if not get_entity_type_instanceof(dataset_entitytype, 'Dataset', auth_header="Bearer " + auth_helper.getProcessSecret()): - return Response(f"{dataset_uuid} is not a dataset will not Publish, entity type is {dataset_entitytype}", 400) + abort_bad_req(f"{dataset_uuid} is not a dataset will not Publish, entity type is {dataset_entitytype}") if not dataset_status == 'QA': - return Response(f"{dataset_uuid} is not in QA state will not Publish, status is {dataset_status}", 400) + abort_bad_req(f"{dataset_uuid} is not in QA state will not Publish, status is {dataset_status}") auth_tokens = auth_helper.getAuthorizationTokens(request.headers) entity_instance = EntitySdk(token=auth_tokens, service_url=current_app.config['ENTITY_WEBSERVICE_URL']) entity = entity_instance.get_entity_by_id(dataset_uuid) - entity_dict: dict = vars(entity) + entity_dict = obj_to_dict(entity) dataset_types_edp = list(Ontology.ops(as_data_dict=True).dataset_types().values()) has_entity_lab_processed_dataset_type: bool = entity_dict.get('dataset_type') in dataset_types_edp @@ -830,17 +836,29 @@ def publish_datastage(identifier): if is_primary or has_entity_lab_processed_dataset_type: if dataset_contacts is None or dataset_contributors is None: - return jsonify({"error": f"{dataset_uuid} missing contacts or contributors. Must have at least one of each"}), 400 + abort_bad_req(f"{dataset_uuid} missing contacts or contributors. Must have at least one of each") dataset_contacts = dataset_contacts.replace("'", '"') dataset_contributors = dataset_contributors.replace("'", '"') if len(json.loads(dataset_contacts)) < 1 or len(json.loads(dataset_contributors)) < 1: - return jsonify({"error": f"{dataset_uuid} missing contacts or contributors. Must have at least one of each"}), 400 + abort_bad_req(f"{dataset_uuid} missing contacts or contributors. Must have at least one of each") + ingest_helper = IngestFileHelper(current_app.config) + ds_path = ingest_helper.dataset_directory_absolute_path(dataset_data_access_level, dataset_group_uuid, dataset_uuid, False) + + md_file = os.path.join(ds_path, "metadata.json") + json_object = entity_json_dumps(entity, auth_tokens, entity_instance) + logger.info(f"publish_datastage; writing metadata.json file: '{md_file}'; containing: '{json_object}'") + try: + with open(md_file, "w") as outfile: + outfile.write(json_object) + except Exception as e: + logger.exception(f"Fatal error while writing md_file {md_file}; {str(e)}") + return jsonify({"error": f"{dataset_uuid} problem writing metadata.json file."}), 500 data_access_level = dataset_data_access_level - #if consortium access level convert to public dataset, if protected access leave it protected + # if consortium access level convert to public dataset, if protected access leave it protected if dataset_data_access_level == 'consortium': - #before moving check to see if there is currently a link for the dataset in the assets directory + # before moving check to see if there is currently a link for the dataset in the assets directory asset_dir = ingest_helper.dataset_asset_directory_absolute_path(dataset_uuid) asset_dir_exists = os.path.exists(asset_dir) ingest_helper.move_dataset_files_for_publishing(dataset_uuid, dataset_group_uuid, 'consortium') @@ -852,28 +870,47 @@ def publish_datastage(identifier): acls_cmd = ingest_helper.set_dataset_permissions(dataset_uuid, dataset_group_uuid, data_access_level, True, no_indexing_and_acls) - auth_tokens = auth_helper.getAuthorizationTokens(request.headers) - entity_instance = EntitySdk(token=auth_tokens, service_url=current_app.config['ENTITY_WEBSERVICE_URL']) - # Generating DOI's for lab processed/derived data as well as IEC/pipeline/airflow processed/derived data). if is_primary or has_entity_lab_processed_dataset_type: # DOI gets generated here # Note: moved dataset title auto generation to entity-api - Zhou 9/29/2021 datacite_doi_helper = DataCiteDoiHelper() - - entity = entity_instance.get_entity_by_id(dataset_uuid) - entity_dict = vars(entity) - try: datacite_doi_helper.create_dataset_draft_doi(entity_dict, check_publication_status=False) except Exception as e: - return jsonify({"error": f"Error occurred while trying to create a draft doi for{dataset_uuid}. {e}"}), 500 + logger.exception(f"Exception while creating a draft doi for {dataset_uuid}: {e}") + return jsonify({"error": f"Error occurred while trying to create a draft doi for {dataset_uuid}. {e}"}), 500 + # This will make the draft DOI created above 'findable'.... try: - datacite_doi_helper.move_doi_state_from_draft_to_findable(entity_dict, auth_tokens) + doi_info = datacite_doi_helper.move_doi_state_from_draft_to_findable(entity_dict, auth_tokens) except Exception as e: - return jsonify({"error": f"Error occurred while trying to change doi draft state to findable doi for{dataset_uuid}. {e}"}), 500 + logger.exception(f"Exception while creating making doi findable and saving to entity for {dataset_uuid}: {e}") + return jsonify({"error": f"Error occurred while trying to change doi draft state to findable doi for {dataset_uuid}. {e}"}), 500 + + doi_update_clause = "" + if doi_info is not None: + doi_update_clause = f", e.registered_doi = '{doi_info['registered_doi']}', e.doi_url = '{doi_info['doi_url']}'" + + # set up a status_history list to add a "Published" entry to below + status_history_list = [] + status_history_str = rval[0].get('status_history') + if status_history_str is not None: + status_history_list = string_helper.convert_str_literal(status_history_str) + + # add Published status change to status history + status_update = { + "status": "Published", + "changed_by_email": user_info['email'], + "change_timestamp": "@#TIMESTAMP#@" + } + status_history_list.append(status_update) + # convert from list to string that is used for storage in database + new_status_history_str = string_helper.convert_py_obj_to_string(status_history_list) + # substitute the TIMESTAMP function to let Neo4j set the change_timestamp value of this status change record + status_history_with_timestamp = new_status_history_str.replace("'@#TIMESTAMP#@'", '" + TIMESTAMP() + "') + status_history_update_clause = f', e.status_history = "{status_history_with_timestamp}"' # set dataset status to published and set the last modified user info and user who published update_q = "match (e:Entity {uuid:'" + dataset_uuid + "'}) set e.status = 'Published', e.last_modified_user_sub = '" + \ @@ -881,12 +918,13 @@ def publish_datastage(identifier): 'email'] + "', e.last_modified_user_displayname = '" + user_info[ 'name'] + "', e.last_modified_timestamp = TIMESTAMP(), e.published_timestamp = TIMESTAMP(), e.published_user_email = '" + \ user_info['email'] + "', e.published_user_sub = '" + user_info[ - 'sub'] + "', e.published_user_displayname = '" + user_info['name'] + "'" + 'sub'] + "', e.published_user_displayname = '" + user_info['name'] + "'" + doi_update_clause + status_history_update_clause + logger.info(dataset_uuid + "\t" + dataset_uuid + "\tNEO4J-update-base-dataset\t" + update_q) neo_session.run(update_q) # triggers a call to entity-api/flush-cache - # out = entity_instance.clear_cache(dataset_uuid) + entity_instance.clear_cache(dataset_uuid) # if all else worked set the list of ids to public that need to be public if len(uuids_for_public) > 0: @@ -894,11 +932,11 @@ def publish_datastage(identifier): update_q = "match (e:Entity) where e.uuid in [" + id_list + "] set e.data_access_level = 'public'" logger.info(identifier + "\t" + dataset_uuid + "\tNEO4J-update-ancestors\t" + update_q) neo_session.run(update_q) - # for e_id in uuids_for_public: - # out = entity_instance.clear_cache(e_id) + for e_id in uuids_for_public: + entity_instance.clear_cache(e_id) if no_indexing_and_acls: - r_val = {'sources_for_indexing': sources_to_reindex} + r_val = {'acl_cmd': acls_cmd, 'sources_for_indexing': sources_to_reindex} else: r_val = {'acl_cmd': '', 'sources_for_indexing': []} @@ -907,7 +945,7 @@ def publish_datastage(identifier): try: rspn = requests.put(current_app.config['SEARCH_WEBSERVICE_URL'] + "/reindex/" + source_uuid, headers={'Authorization': request.headers["AUTHORIZATION"]}) logger.info(f"Publishing {identifier} indexed source {source_uuid} with status {rspn.status_code}") - except: + except Exception: logger.exception(f"While publishing {identifier} Error happened when calling reindex web service for source {source_uuid}") return Response(json.dumps(r_val), 200, mimetype='application/json') @@ -1716,3 +1754,32 @@ def get_entity_type_instanceof(type_a, type_b, auth_header=None) -> bool: resp_json: dict = response.json() return resp_json['instanceof'] + + +def obj_to_dict(obj) -> dict: + """ + Convert the obj[ect] into a dict, but deeply. + Note: The Python builtin 'vars()' does not work here because of the way that some of the classes + are defined. + """ + return json.loads( + json.dumps(obj, default=lambda o: getattr(o, '__dict__', str(o))) + ) + + +def entity_json_dumps(entity: Entity, token: str, entity_sdk: EntitySdk) -> str: + """ + Because entity and the content of the arrays returned from entity_instance.get_associated_* + contain user defined objects we need to turn them into simple python objects (e.g., dicts, lists, str) + before we can convert them wth json.dumps. + Here we create an expanded version of the entity associated with the dataset_uuid and return it as a json string. + """ + dataset_uuid = entity.get_uuid() + entity = obj_to_dict(entity) + entity['organs'] = obj_to_dict(entity_sdk.get_associated_organs_from_dataset(dataset_uuid)) + entity['samples'] = obj_to_dict(entity_sdk.get_associated_samples_from_dataset(dataset_uuid)) + entity['sources'] = get_associated_sources_from_dataset(dataset_uuid, token=token, as_dict=True) + + json_object = json.dumps(entity, indent=4) + json_object += '\n' + return json_object diff --git a/src/routes/entity_CRUD/ingest_file_helper.py b/src/routes/entity_CRUD/ingest_file_helper.py index ce76c327..c9bf8fc2 100644 --- a/src/routes/entity_CRUD/ingest_file_helper.py +++ b/src/routes/entity_CRUD/ingest_file_helper.py @@ -40,9 +40,9 @@ def get_dataset_directory_absolute_path(self, dataset_record, group_uuid, datase if 'status' in dataset_record and dataset_record['status'] == 'Published': published = True - return self.__dataset_directory_absolute_path(access_level, group_uuid, dataset_uuid, published) + return self.dataset_directory_absolute_path(access_level, group_uuid, dataset_uuid, published) - def __dataset_directory_absolute_path(self, access_level, group_uuid, dataset_uuid, published): + def dataset_directory_absolute_path(self, access_level, group_uuid, dataset_uuid, published): grp_name = AuthHelper.getGroupDisplayName(group_uuid) if access_level == 'protected': base_dir = self.appconfig['GLOBUS_PROTECTED_ENDPOINT_FILEPATH'] @@ -175,12 +175,12 @@ def set_dir_permissions(self, access_level, file_path, published=False, trial_ru self.logger.error(e, exc_info=True) def move_dataset_files_for_publishing(self, uuid, group_uuid, dataset_access_level, trial_run=False): - from_path = self.__dataset_directory_absolute_path(dataset_access_level, group_uuid, uuid, False) + from_path = self.dataset_directory_absolute_path(dataset_access_level, group_uuid, uuid, False) if not os.path.isdir(from_path): raise HTTPException(f"{uuid}: path not found to dataset will not publish, path is {from_path}", 500) data_access_level = 'protected' if not dataset_access_level == 'protected': data_access_level = 'public' - to_path = self.__dataset_directory_absolute_path(data_access_level, group_uuid, uuid, True) + to_path = self.dataset_directory_absolute_path(data_access_level, group_uuid, uuid, True) if not trial_run: shutil.move(from_path, to_path) else: @@ -188,7 +188,6 @@ def move_dataset_files_for_publishing(self, uuid, group_uuid, dataset_access_lev return None - def get_upload_directory_absolute_path(self, group_uuid, upload_uuid): grp_name = AuthHelper.getGroupDisplayName(group_uuid) base_dir = self.appconfig['GLOBUS_PROTECTED_ENDPOINT_FILEPATH'] @@ -204,9 +203,8 @@ def create_upload_directory(self, group_uuid, upload_uuid): except Exception as e: self.logger.error(e, exc_info=True) - def set_dataset_permissions(self, dataset_uuid, group_uuid, dataset_access_level, published, trial_run=False): - file_path = self.__dataset_directory_absolute_path(dataset_access_level, group_uuid, dataset_uuid, published) + file_path = self.dataset_directory_absolute_path(dataset_access_level, group_uuid, dataset_uuid, published) return self.set_dir_permissions(dataset_access_level, file_path, published, trial_run=trial_run) def relink_to_public(self, dataset_uuid):