Skip to content

Commit

Permalink
[ENH] Add function to share xnat sessions. Fix bugs, update template
Browse files Browse the repository at this point in the history
  • Loading branch information
@DESm1th
DESm1th committed Nov 6, 2024
1 parent c9d0321 commit 4bc5cf1
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 27 deletions.
12 changes: 12 additions & 0 deletions assets/config_templates/main_config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,13 @@ ExportSettings:
# metadata folder that will hold the username
# and password for the XnatSource server.
# Must be provided if XnatSource is set.
# XnatDataSharing: <boolean> # Whether or not certain XNAT sessions for
# a study will be shared within XNAT. Any
# value is interpreted as 'True', if unset
# will be False. Shared IDs are read from
# the configured redcap survey (i.e. you
# should also set 'RedcapSharedIdPrefix' if
# you set this).


###### REDCap Configuration ##############
Expand Down Expand Up @@ -171,6 +178,11 @@ ExportSettings:
# RedcapRecordKey: <fieldname> # The name of the field that contains the
# unique record ID.
# RedcapComments: <fieldname> # The name of field that holds RA comments.
# RedcapSharedIdPrefix: <prefix> # The prefix that will be used for any
# survey field that contains an alternate
# ID for the session. Optional. If XNAT
# sessions must be shared then
# 'XnatDataSharing' must be set as well.


###### Log Server Configuration ##############
Expand Down
116 changes: 99 additions & 17 deletions bin/dm_link_shared_ids.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import datman.config
import datman.scanid
import datman.utils
import datman.xnat

import bin.dm_link_project_scans as link_scans
import datman.dashboard as dashboard
Expand Down Expand Up @@ -70,12 +71,16 @@ def main():
link_scans.logger.setLevel(logging.ERROR)

config = datman.config.config(filename=site_config, study=project)
if uses_xnat_sharing(config):
xnat = datman.xnat.get_connection(config)
else:
xnat = None

scan_complete_records = get_redcap_records(config, redcap_cred)
for record in scan_complete_records:
if not record.shared_ids:
continue
make_links(record)
share_session(record, xnat_connection=xnat)


def get_redcap_records(config, redcap_cred):
Expand All @@ -85,9 +90,8 @@ def get_redcap_records(config, redcap_cred):

record_id_field = get_setting(config, 'RedcapRecordKey')
subid_field = get_setting(config, 'RedcapSubj', default='par_id')
comment_field = get_setting(config, 'RedcapComments', default='cmts')
shared_prefix_field = get_setting(
config, 'RedcapSharedIdPrefix', default='shared_id')
config, 'RedcapSharedIdPrefix', default='shared_parid')

try:
id_map = config.get_key('IdMap')
Expand All @@ -109,7 +113,7 @@ def get_redcap_records(config, redcap_cred):
try:
project_records = parse_records(
response, current_study, id_map,
record_id_field, subid_field, comment_field, shared_prefix_field)
record_id_field, subid_field, shared_prefix_field)
except ValueError as e:
logger.error("Couldn't parse redcap records for server response {}. "
"Reason: {}".format(response.content, e))
Expand Down Expand Up @@ -146,11 +150,11 @@ def get_setting(config, key, default=None):


def parse_records(response, study, id_map, record_id_field, id_field,
comment_field, shared_id_prefix_field):
shared_id_prefix_field):
records = []
for item in response.json():
record = Record(item, id_map, record_id_field=record_id_field,
id_field=id_field, comment_field=comment_field,
id_field=id_field,
shared_id_prefix_field=shared_id_prefix_field)
if record.id is None:
logger.debug(
Expand All @@ -163,13 +167,24 @@ def parse_records(response, study, id_map, record_id_field, id_field,
return records


def make_links(record):
def uses_xnat_sharing(config):
"""Check if source (xnat) data is to be shared also.
"""
try:
config.get_key('XnatDataSharing')
except datman.config.UndefinedSetting:
return False
return True


def share_session(record, xnat_connection=None):
source = record.id
for target in record.shared_ids:
logger.info(
f"Making links from source {source} to target {target}"
f"Sharing source ID {source} under ID {target}"
)
target_cfg = datman.config.config(study=target)

target_cfg = datman.config.config(study=str(target))
try:
target_tags = list(target_cfg.get_tags(site=source.site))
except Exception:
Expand All @@ -179,14 +194,74 @@ def make_links(record):

if DRYRUN:
logger.info(
"DRYRUN - would have made links from source ID "
"DRYRUN - would have shared scans from source ID "
f"{source} to target ID {target} for tags {target_tags}"
)
continue

if xnat_connection:
share_xnat_data(xnat_connection, source, target, target_cfg)

link_scans.create_linked_session(str(source), str(target), target_tags)

if dashboard.dash_found:
share_redcap_record(target, record)
share_redcap_record(str(target), record)


def share_xnat_data(xnat_connection, source, dest, config):
"""Share an xnat subject/experiment under another ID.
Args:
xnat_connection (:obj:`datman.xnat.xnat`): A connection to the xnat
server containing the source data.
source (:obj:`datman.scanid.Identifier`): A datman Identifier for
the original session data.
dest (:obj:`datman.scanid.Identifier`): A datman Identifier for the
destination ID to share the data under.
config (:obj:`datman.config.config`): A datman configuration object.
Raises:
requests.HTTPError: If issues arise while communicating with the
server.
"""
source_project = xnat_connection.find_project(
source.get_xnat_subject_id(),
config.get_xnat_projects(str(source))
)
dest_project = xnat_connection.find_project(
dest.get_xnat_subject_id(),
config.get_xnat_projects(str(dest))
)

try:
xnat_connection.share_subject(
source_project,
source.get_xnat_subject_id(),
dest_project,
dest.get_xnat_subject_id()
)
except datman.xnat.XnatException:
logger.info(f"XNAT subject {dest.get_xnat_subject_id()}"
" already exists. No changes made.")
except requests.HTTPError as e:
logger.error(f"Failed while sharing source ID {source} into dest ID "
f"{dest} on XNAT. Reason - {e}")
return

try:
xnat_connection.share_experiment(
source_project,
source.get_xnat_subject_id(),
source.get_xnat_experiment_id(),
dest_project,
dest.get_xnat_experiment_id()
)
except datman.xnat.XnatException:
logger.info(f"XNAT experiment {dest.get_xnat_experiment_id()}"
" already exists. No changes made.")
except requests.HTTPError as e:
logger.error(f"Failed while sharing source ID {source} into dest ID "
f"{dest} on XNAT. Reason - {e}")


def share_redcap_record(session, shared_record):
Expand Down Expand Up @@ -223,12 +298,19 @@ def share_redcap_record(session, shared_record):

class Record(object):
def __init__(self, record_dict, id_map=None, record_id_field='record_id',
id_field='par_id', comment_field='cmts',
shared_id_prefix_field='shared_parid'):
self.record_id = record_dict[record_id_field]
self.id = self.__get_datman_id(record_dict[id_field], id_map)
id_field='par_id', shared_id_prefix_field='shared_parid'):
try:
self.record_id = record_dict[record_id_field]
except KeyError:
# Try default in case user misconfigured their fields!
self.record_id = record_dict['record_id']
try:
subid = record_dict[id_field]
except KeyError:
# Try default in case user misconfigured their fields!
subid = record_dict['par_id']
self.id = self.__get_datman_id(subid, id_map)
self.study = self.__get_study()
self.comment = record_dict[comment_field]
self.shared_ids = self.__get_shared_ids(record_dict, id_map,
shared_id_prefix_field)

Expand Down Expand Up @@ -259,7 +341,7 @@ def __get_shared_ids(self, record_dict, id_map, shared_id_prefix_field):
if subject_id is None:
# Badly named shared id value. Skip it.
continue
shared_ids.append(str(subject_id))
shared_ids.append(subject_id)
return shared_ids

def __get_datman_id(self, subid, id_map):
Expand Down
63 changes: 53 additions & 10 deletions tests/test_dm_link_shared_ids.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,13 @@ class TestRecord(unittest.TestCase):
'record_id': 0,
'shared_parid_1': 'STUDY_SITE_0002_01_01',
'shared_parid_2': 'STUDY2_CMH_9999_01_01',
'shared_parid_8': 'OTHER_CMH_1234_01_01',
'cmts': 'No comment.'}
'shared_parid_8': 'OTHER_CMH_1234_01_01'}
mock_kcni_record = {'par_id': 'STU01_ABC_0001_01_SE01_MR',
'record_id': 1,
'shared_parid_1': 'STU02_ABC_0002_01_SE01_MR',
'shared_parid_2': 'STUDY3_ABC_0003_01_SE01_MR',
'cmts': 'Test comment.'}
'shared_parid_2': 'STUDY3_ABC_0003_01_SE01_MR'}
mock_diff_fields_record = {'mri_sub_id': 'STU01_ABC_0001_01_SE01_MR',
'id': 2,
'mri_cmts': 'Testing non-standard field names!',
'shared_id': 'STU02_DEF_0002_04_SE01_MR'}

def test_ignores_records_with_bad_subject_id(self):
Expand All @@ -45,7 +42,7 @@ def test_ignores_badly_named_shared_ids(self):

record = link_shared.Record(bad_shared_id)

assert bad_id not in record.shared_ids
assert bad_id not in [str(item) for item in record.shared_ids]

def test_finds_all_shared_ids_in_record(self):
record = link_shared.Record(self.mock_redcap_record)
Expand All @@ -54,7 +51,8 @@ def test_finds_all_shared_ids_in_record(self):
self.mock_redcap_record['shared_parid_2'],
self.mock_redcap_record['shared_parid_8']]

assert sorted(record.shared_ids) == sorted(expected)
actual_ids = [str(item) for item in record.shared_ids]
assert sorted(actual_ids) == sorted(expected)

def test_correctly_handles_kcni_main_id(self):
id_map = {
Expand Down Expand Up @@ -82,7 +80,8 @@ def test_correctly_handles_kcni_shared_ids(self):

record = link_shared.Record(self.mock_kcni_record, id_map)

assert 'STUDY2_SITE_0002_01_01' in record.shared_ids
shared_ids = [str(item) for item in record.shared_ids]
assert 'STUDY2_SITE_0002_01_01' in shared_ids

def test_handles_nonstandard_field_names(self):
id_map = {
Expand All @@ -101,8 +100,52 @@ def test_handles_nonstandard_field_names(self):
id_map,
record_id_field='id',
id_field='mri_sub_id',
comment_field='mri_cmts',
shared_id_prefix_field='shared_id')

assert str(record.id) == 'STUDY_SITE_0001_01_01'
assert 'STUDY2_SITE2_0002_04_01' in record.shared_ids
shared_ids = [str(item) for item in record.shared_ids]
assert 'STUDY2_SITE2_0002_04_01' in shared_ids

def test_uses_default_for_misconfigured_record_id_field(self):
id_map = {
'Study': {
'STU01': 'STUDY',
'STU02': 'STUDY2'
},
'Site': {
'ABC': 'SITE'
}
}

record = link_shared.Record(
self.mock_kcni_record,
id_map,
record_id_field='bad_record_id_field'
)

assert str(record.id) == 'STUDY_SITE_0001_01_01'

shared_ids = [str(item) for item in record.shared_ids]
assert 'STUDY2_SITE_0002_01_01' in shared_ids

def test_uses_default_for_misconfigured_id_field(self):
id_map = {
'Study': {
'STU01': 'STUDY',
'STU02': 'STUDY2'
},
'Site': {
'ABC': 'SITE'
}
}

record = link_shared.Record(
self.mock_kcni_record,
id_map,
id_field='bad_id_field'
)

assert str(record.id) == 'STUDY_SITE_0001_01_01'

shared_ids = [str(item) for item in record.shared_ids]
assert 'STUDY2_SITE_0002_01_01' in shared_ids

0 comments on commit 4bc5cf1

Please sign in to comment.