From 4bc5cf1a5a3ac1678cdd2a9ff022bc63fe8f9a62 Mon Sep 17 00:00:00 2001 From: Dawn Smith Date: Wed, 6 Nov 2024 18:11:57 -0500 Subject: [PATCH] [ENH] Add function to share xnat sessions. Fix bugs, update template --- assets/config_templates/main_config.yml | 12 +++ bin/dm_link_shared_ids.py | 116 ++++++++++++++++++++---- tests/test_dm_link_shared_ids.py | 63 +++++++++++-- 3 files changed, 164 insertions(+), 27 deletions(-) diff --git a/assets/config_templates/main_config.yml b/assets/config_templates/main_config.yml index 52ca50eb..f67ed9fe 100644 --- a/assets/config_templates/main_config.yml +++ b/assets/config_templates/main_config.yml @@ -140,6 +140,13 @@ ExportSettings: # metadata folder that will hold the username # and password for the XnatSource server. # Must be provided if XnatSource is set. +# XnatDataSharing: # Whether or not certain XNAT sessions for + # a study will be shared within XNAT. Any + # value is interpreted as 'True', if unset + # will be False. Shared IDs are read from + # the configured redcap survey (i.e. you + # should also set 'RedcapSharedIdPrefix' if + # you set this). ###### REDCap Configuration ############## @@ -171,6 +178,11 @@ ExportSettings: # RedcapRecordKey: # The name of the field that contains the # unique record ID. # RedcapComments: # The name of field that holds RA comments. +# RedcapSharedIdPrefix: # The prefix that will be used for any + # survey field that contains an alternate + # ID for the session. Optional. If XNAT + # sessions must be shared then + # 'XnatDataSharing' must be set as well. ###### Log Server Configuration ############## diff --git a/bin/dm_link_shared_ids.py b/bin/dm_link_shared_ids.py index e9cd25a2..1773f248 100755 --- a/bin/dm_link_shared_ids.py +++ b/bin/dm_link_shared_ids.py @@ -34,6 +34,7 @@ import datman.config import datman.scanid import datman.utils +import datman.xnat import bin.dm_link_project_scans as link_scans import datman.dashboard as dashboard @@ -70,12 +71,16 @@ def main(): link_scans.logger.setLevel(logging.ERROR) config = datman.config.config(filename=site_config, study=project) + if uses_xnat_sharing(config): + xnat = datman.xnat.get_connection(config) + else: + xnat = None scan_complete_records = get_redcap_records(config, redcap_cred) for record in scan_complete_records: if not record.shared_ids: continue - make_links(record) + share_session(record, xnat_connection=xnat) def get_redcap_records(config, redcap_cred): @@ -85,9 +90,8 @@ def get_redcap_records(config, redcap_cred): record_id_field = get_setting(config, 'RedcapRecordKey') subid_field = get_setting(config, 'RedcapSubj', default='par_id') - comment_field = get_setting(config, 'RedcapComments', default='cmts') shared_prefix_field = get_setting( - config, 'RedcapSharedIdPrefix', default='shared_id') + config, 'RedcapSharedIdPrefix', default='shared_parid') try: id_map = config.get_key('IdMap') @@ -109,7 +113,7 @@ def get_redcap_records(config, redcap_cred): try: project_records = parse_records( response, current_study, id_map, - record_id_field, subid_field, comment_field, shared_prefix_field) + record_id_field, subid_field, shared_prefix_field) except ValueError as e: logger.error("Couldn't parse redcap records for server response {}. " "Reason: {}".format(response.content, e)) @@ -146,11 +150,11 @@ def get_setting(config, key, default=None): def parse_records(response, study, id_map, record_id_field, id_field, - comment_field, shared_id_prefix_field): + shared_id_prefix_field): records = [] for item in response.json(): record = Record(item, id_map, record_id_field=record_id_field, - id_field=id_field, comment_field=comment_field, + id_field=id_field, shared_id_prefix_field=shared_id_prefix_field) if record.id is None: logger.debug( @@ -163,13 +167,24 @@ def parse_records(response, study, id_map, record_id_field, id_field, return records -def make_links(record): +def uses_xnat_sharing(config): + """Check if source (xnat) data is to be shared also. + """ + try: + config.get_key('XnatDataSharing') + except datman.config.UndefinedSetting: + return False + return True + + +def share_session(record, xnat_connection=None): source = record.id for target in record.shared_ids: logger.info( - f"Making links from source {source} to target {target}" + f"Sharing source ID {source} under ID {target}" ) - target_cfg = datman.config.config(study=target) + + target_cfg = datman.config.config(study=str(target)) try: target_tags = list(target_cfg.get_tags(site=source.site)) except Exception: @@ -179,14 +194,74 @@ def make_links(record): if DRYRUN: logger.info( - "DRYRUN - would have made links from source ID " + "DRYRUN - would have shared scans from source ID " f"{source} to target ID {target} for tags {target_tags}" ) continue + if xnat_connection: + share_xnat_data(xnat_connection, source, target, target_cfg) + link_scans.create_linked_session(str(source), str(target), target_tags) + if dashboard.dash_found: - share_redcap_record(target, record) + share_redcap_record(str(target), record) + + +def share_xnat_data(xnat_connection, source, dest, config): + """Share an xnat subject/experiment under another ID. + + Args: + xnat_connection (:obj:`datman.xnat.xnat`): A connection to the xnat + server containing the source data. + source (:obj:`datman.scanid.Identifier`): A datman Identifier for + the original session data. + dest (:obj:`datman.scanid.Identifier`): A datman Identifier for the + destination ID to share the data under. + config (:obj:`datman.config.config`): A datman configuration object. + + Raises: + requests.HTTPError: If issues arise while communicating with the + server. + """ + source_project = xnat_connection.find_project( + source.get_xnat_subject_id(), + config.get_xnat_projects(str(source)) + ) + dest_project = xnat_connection.find_project( + dest.get_xnat_subject_id(), + config.get_xnat_projects(str(dest)) + ) + + try: + xnat_connection.share_subject( + source_project, + source.get_xnat_subject_id(), + dest_project, + dest.get_xnat_subject_id() + ) + except datman.xnat.XnatException: + logger.info(f"XNAT subject {dest.get_xnat_subject_id()}" + " already exists. No changes made.") + except requests.HTTPError as e: + logger.error(f"Failed while sharing source ID {source} into dest ID " + f"{dest} on XNAT. Reason - {e}") + return + + try: + xnat_connection.share_experiment( + source_project, + source.get_xnat_subject_id(), + source.get_xnat_experiment_id(), + dest_project, + dest.get_xnat_experiment_id() + ) + except datman.xnat.XnatException: + logger.info(f"XNAT experiment {dest.get_xnat_experiment_id()}" + " already exists. No changes made.") + except requests.HTTPError as e: + logger.error(f"Failed while sharing source ID {source} into dest ID " + f"{dest} on XNAT. Reason - {e}") def share_redcap_record(session, shared_record): @@ -223,12 +298,19 @@ def share_redcap_record(session, shared_record): class Record(object): def __init__(self, record_dict, id_map=None, record_id_field='record_id', - id_field='par_id', comment_field='cmts', - shared_id_prefix_field='shared_parid'): - self.record_id = record_dict[record_id_field] - self.id = self.__get_datman_id(record_dict[id_field], id_map) + id_field='par_id', shared_id_prefix_field='shared_parid'): + try: + self.record_id = record_dict[record_id_field] + except KeyError: + # Try default in case user misconfigured their fields! + self.record_id = record_dict['record_id'] + try: + subid = record_dict[id_field] + except KeyError: + # Try default in case user misconfigured their fields! + subid = record_dict['par_id'] + self.id = self.__get_datman_id(subid, id_map) self.study = self.__get_study() - self.comment = record_dict[comment_field] self.shared_ids = self.__get_shared_ids(record_dict, id_map, shared_id_prefix_field) @@ -259,7 +341,7 @@ def __get_shared_ids(self, record_dict, id_map, shared_id_prefix_field): if subject_id is None: # Badly named shared id value. Skip it. continue - shared_ids.append(str(subject_id)) + shared_ids.append(subject_id) return shared_ids def __get_datman_id(self, subid, id_map): diff --git a/tests/test_dm_link_shared_ids.py b/tests/test_dm_link_shared_ids.py index 318be9d9..3b7d4481 100644 --- a/tests/test_dm_link_shared_ids.py +++ b/tests/test_dm_link_shared_ids.py @@ -14,16 +14,13 @@ class TestRecord(unittest.TestCase): 'record_id': 0, 'shared_parid_1': 'STUDY_SITE_0002_01_01', 'shared_parid_2': 'STUDY2_CMH_9999_01_01', - 'shared_parid_8': 'OTHER_CMH_1234_01_01', - 'cmts': 'No comment.'} + 'shared_parid_8': 'OTHER_CMH_1234_01_01'} mock_kcni_record = {'par_id': 'STU01_ABC_0001_01_SE01_MR', 'record_id': 1, 'shared_parid_1': 'STU02_ABC_0002_01_SE01_MR', - 'shared_parid_2': 'STUDY3_ABC_0003_01_SE01_MR', - 'cmts': 'Test comment.'} + 'shared_parid_2': 'STUDY3_ABC_0003_01_SE01_MR'} mock_diff_fields_record = {'mri_sub_id': 'STU01_ABC_0001_01_SE01_MR', 'id': 2, - 'mri_cmts': 'Testing non-standard field names!', 'shared_id': 'STU02_DEF_0002_04_SE01_MR'} def test_ignores_records_with_bad_subject_id(self): @@ -45,7 +42,7 @@ def test_ignores_badly_named_shared_ids(self): record = link_shared.Record(bad_shared_id) - assert bad_id not in record.shared_ids + assert bad_id not in [str(item) for item in record.shared_ids] def test_finds_all_shared_ids_in_record(self): record = link_shared.Record(self.mock_redcap_record) @@ -54,7 +51,8 @@ def test_finds_all_shared_ids_in_record(self): self.mock_redcap_record['shared_parid_2'], self.mock_redcap_record['shared_parid_8']] - assert sorted(record.shared_ids) == sorted(expected) + actual_ids = [str(item) for item in record.shared_ids] + assert sorted(actual_ids) == sorted(expected) def test_correctly_handles_kcni_main_id(self): id_map = { @@ -82,7 +80,8 @@ def test_correctly_handles_kcni_shared_ids(self): record = link_shared.Record(self.mock_kcni_record, id_map) - assert 'STUDY2_SITE_0002_01_01' in record.shared_ids + shared_ids = [str(item) for item in record.shared_ids] + assert 'STUDY2_SITE_0002_01_01' in shared_ids def test_handles_nonstandard_field_names(self): id_map = { @@ -101,8 +100,52 @@ def test_handles_nonstandard_field_names(self): id_map, record_id_field='id', id_field='mri_sub_id', - comment_field='mri_cmts', shared_id_prefix_field='shared_id') assert str(record.id) == 'STUDY_SITE_0001_01_01' - assert 'STUDY2_SITE2_0002_04_01' in record.shared_ids + shared_ids = [str(item) for item in record.shared_ids] + assert 'STUDY2_SITE2_0002_04_01' in shared_ids + + def test_uses_default_for_misconfigured_record_id_field(self): + id_map = { + 'Study': { + 'STU01': 'STUDY', + 'STU02': 'STUDY2' + }, + 'Site': { + 'ABC': 'SITE' + } + } + + record = link_shared.Record( + self.mock_kcni_record, + id_map, + record_id_field='bad_record_id_field' + ) + + assert str(record.id) == 'STUDY_SITE_0001_01_01' + + shared_ids = [str(item) for item in record.shared_ids] + assert 'STUDY2_SITE_0002_01_01' in shared_ids + + def test_uses_default_for_misconfigured_id_field(self): + id_map = { + 'Study': { + 'STU01': 'STUDY', + 'STU02': 'STUDY2' + }, + 'Site': { + 'ABC': 'SITE' + } + } + + record = link_shared.Record( + self.mock_kcni_record, + id_map, + id_field='bad_id_field' + ) + + assert str(record.id) == 'STUDY_SITE_0001_01_01' + + shared_ids = [str(item) for item in record.shared_ids] + assert 'STUDY2_SITE_0002_01_01' in shared_ids