From 9cd1ea149f091d9a0de67cba8f3e52635fe02919 Mon Sep 17 00:00:00 2001 From: bendichter Date: Wed, 13 Apr 2022 12:58:55 -0400 Subject: [PATCH 01/14] add RemoteDandiset.is_nwb() convenience function --- dandi/dandiapi.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 2b6c79a90..1e4e73ce9 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1071,6 +1071,17 @@ def iter_upload_raw_asset( self, metadata=asset_metadata, jobs=jobs, replacing=replace_asset ) + def is_nwb(self): + assetsSummary = self.get_raw_metadata()["assetsSummary"] + if "dataStandard" not in assetsSummary: + return False + return any( + [ + x["identifier"] == "RRID:SCR_015242" + for x in assetsSummary["dataStandard"] + ] + ) + class BaseRemoteAsset(ABC, APIBase): """ From 27b0ce5d2cdbdebd90bf3a1034c5fdee6ff0e467 Mon Sep 17 00:00:00 2001 From: bendichter Date: Wed, 13 Apr 2022 17:02:00 -0400 Subject: [PATCH 02/14] add docstring and remove unnecessary [] --- dandi/dandiapi.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 1e4e73ce9..b8be10069 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1072,14 +1072,17 @@ def iter_upload_raw_asset( ) def is_nwb(self): + """ + Returns True if the dandiset contains NWB file assets. + + This is determined by checking for "RRID:SCR_015242" in the "dataStandard" field + of the assetsSummary of the dandiset. + """ assetsSummary = self.get_raw_metadata()["assetsSummary"] if "dataStandard" not in assetsSummary: return False return any( - [ - x["identifier"] == "RRID:SCR_015242" - for x in assetsSummary["dataStandard"] - ] + x["identifier"] == "RRID:SCR_015242" for x in assetsSummary["dataStandard"] ) From 21261b1493287ea89b7d512865dc9102c5f6c7b9 Mon Sep 17 00:00:00 2001 From: bendichter Date: Wed, 13 Apr 2022 17:04:15 -0400 Subject: [PATCH 03/14] add type annotation --- dandi/dandiapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index b8be10069..b034ecdbc 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1071,7 +1071,7 @@ def iter_upload_raw_asset( self, metadata=asset_metadata, jobs=jobs, replacing=replace_asset ) - def is_nwb(self): + def is_nwb(self) -> bool: """ Returns True if the dandiset contains NWB file assets. From 74b374912761d84072bc94cde4fbaee3a0ca84cc Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 13 Apr 2022 17:06:18 -0400 Subject: [PATCH 04/14] Update dandi/dandiapi.py Co-authored-by: John T. Wodder II --- dandi/dandiapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index b034ecdbc..e3680f230 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1073,7 +1073,7 @@ def iter_upload_raw_asset( def is_nwb(self) -> bool: """ - Returns True if the dandiset contains NWB file assets. + Returns True if the Dandiset contains one or more NWB file assets. This is determined by checking for "RRID:SCR_015242" in the "dataStandard" field of the assetsSummary of the dandiset. From 5bc0f54a1144f9fe97ea27f7c34d7fee548e801a Mon Sep 17 00:00:00 2001 From: bendichter Date: Mon, 31 Oct 2022 14:51:03 -0400 Subject: [PATCH 05/14] generalize method to has_data_standard --- dandi/dandiapi.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index e3680f230..b95eb7f14 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -60,6 +60,8 @@ T = TypeVar("T") +DATA_STANDARD_MAP = dict(NWB="RRID:SCR_015242") + class AssetType(Enum): """ @@ -1071,19 +1073,24 @@ def iter_upload_raw_asset( self, metadata=asset_metadata, jobs=jobs, replacing=replace_asset ) - def is_nwb(self) -> bool: + def has_data_standard(self, data_standard: str) -> bool: """ - Returns True if the Dandiset contains one or more NWB file assets. - - This is determined by checking for "RRID:SCR_015242" in the "dataStandard" field - of the assetsSummary of the dandiset. + Returns True if the Dandiset contains one or more files of the indicated + standard. Otherwise, returns False. """ - assetsSummary = self.get_raw_metadata()["assetsSummary"] - if "dataStandard" not in assetsSummary: + if data_standard in DATA_STANDARD_MAP: + rrid = DATA_STANDARD_MAP[data_standard] + elif data_standard.startswith("RRID:"): + rrid = data_standard + else: + raise ValueError( + f"'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one " + f"of the following values: {DATA_STANDARD_MAP.keys()}" + ) + assets_summary = self.get_raw_metadata()["assetsSummary"] + if "dataStandard" not in assets_summary: return False - return any( - x["identifier"] == "RRID:SCR_015242" for x in assetsSummary["dataStandard"] - ) + return any(x["identifier"] == rrid for x in assets_summary["dataStandard"]) class BaseRemoteAsset(ABC, APIBase): From 2b008ef7fe874c27e34638e80a62b9510ccff89b Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Mon, 31 Oct 2022 21:02:16 -0400 Subject: [PATCH 06/14] Update dandi/dandiapi.py --- dandi/dandiapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index b95eb7f14..aa21d1cde 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1085,7 +1085,7 @@ def has_data_standard(self, data_standard: str) -> bool: else: raise ValueError( f"'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one " - f"of the following values: {DATA_STANDARD_MAP.keys()}" + f"of the following values: {", ".join(DATA_STANDARD_MAP.keys())}" ) assets_summary = self.get_raw_metadata()["assetsSummary"] if "dataStandard" not in assets_summary: From fce70ebe0ef78d25e77996cc7ee73ba80ebede93 Mon Sep 17 00:00:00 2001 From: bendichter Date: Mon, 31 Oct 2022 22:54:27 -0400 Subject: [PATCH 07/14] better docstring --- dandi/dandiapi.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index b95eb7f14..c6b24633e 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -60,7 +60,10 @@ T = TypeVar("T") -DATA_STANDARD_MAP = dict(NWB="RRID:SCR_015242") +DATA_STANDARD_MAP = dict( + NWB="RRID:SCR_015242", + BIDS="RRID:SCR_016124", +) class AssetType(Enum): @@ -1076,7 +1079,13 @@ def iter_upload_raw_asset( def has_data_standard(self, data_standard: str) -> bool: """ Returns True if the Dandiset contains one or more files of the indicated - standard. Otherwise, returns False. + standard. Otherwise, returns False. This is determined by checking for + the RRID of the standard in the "dataStandard" field of the assetsSummary of + the dandiset. + + :param data_standard: str that can either be "NWB", "BIDS", or an RRID of a + standard. + :type data_standard: str """ if data_standard in DATA_STANDARD_MAP: rrid = DATA_STANDARD_MAP[data_standard] From b1aabee7b58bee550ab43c30fe32c7d843a73748 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 1 Nov 2022 12:40:09 -0400 Subject: [PATCH 08/14] Update dandi/dandiapi.py Co-authored-by: Yaroslav Halchenko --- dandi/dandiapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 9a3324128..38d9c7c98 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1093,7 +1093,7 @@ def has_data_standard(self, data_standard: str) -> bool: rrid = data_standard else: raise ValueError( - f"'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one " + "'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one " f"of the following values: {", ".join(DATA_STANDARD_MAP.keys())}" ) assets_summary = self.get_raw_metadata()["assetsSummary"] From 203d4ad604c4199015deb3b5a86f85a0a94e95df Mon Sep 17 00:00:00 2001 From: bendichter Date: Tue, 7 Mar 2023 21:34:21 -0300 Subject: [PATCH 09/14] fix string rendering and add tests --- dandi/dandiapi.py | 5 ++--- dandi/tests/test_dandiapi.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 38d9c7c98..32402d8e4 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1083,8 +1083,7 @@ def has_data_standard(self, data_standard: str) -> bool: the RRID of the standard in the "dataStandard" field of the assetsSummary of the dandiset. - :param data_standard: str that can either be "NWB", "BIDS", or an RRID of a - standard. + :param data_standard: can be "NWB", "BIDS", or an RRID of a standard. :type data_standard: str """ if data_standard in DATA_STANDARD_MAP: @@ -1094,7 +1093,7 @@ def has_data_standard(self, data_standard: str) -> bool: else: raise ValueError( "'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one " - f"of the following values: {", ".join(DATA_STANDARD_MAP.keys())}" + f"of the following values: {', '.join(DATA_STANDARD_MAP.keys())}" ) assets_summary = self.get_raw_metadata()["assetsSummary"] if "dataStandard" not in assets_summary: diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 52d179fbb..e2809530e 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -635,3 +635,20 @@ def test_empty_zarr_iterfiles(new_dandiset: SampleDandiset) -> None: a = RemoteAsset.from_data(new_dandiset.dandiset, r) assert isinstance(a, RemoteZarrAsset) assert list(a.iterfiles()) == [] + + +def test_dandiset_has_data_standard(): + with DandiAPIClient() as client: + dandiset = client.get_dandiset("000003", version_id='0.210812.1448') + assert dandiset.has_data_standard("NWB") + assert dandiset.has_data_standard("RRID:SCR_015242") + assert not dandiset.has_data_standard("RRID:XXX_000000") + assert not dandiset.has_data_standard("BIDS") + + +def test_dandiset_has_data_standard_incorrect_arg(): + with DandiAPIClient() as client: + dandiset = client.get_dandiset("000003", version_id='0.210812.1448') + with pytest.raises(ValueError) as exc_info: + dandiset.has_data_standard("NWC") + assert str(exc_info.value) == "'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one of the following values: NWB, BIDS" From 6eea7c41039c21865a593f3901fd95dfc1303712 Mon Sep 17 00:00:00 2001 From: bendichter Date: Fri, 10 Mar 2023 10:44:53 -0300 Subject: [PATCH 10/14] Trigger Build From d2073c26f8e1b527e326f64cc4fd360b93ebc9d2 Mon Sep 17 00:00:00 2001 From: bendichter Date: Fri, 10 Mar 2023 10:52:42 -0300 Subject: [PATCH 11/14] Trigger Build From fd5cdd565f0bfd10faabba5255e9fc59e4c1d699 Mon Sep 17 00:00:00 2001 From: bendichter Date: Fri, 10 Mar 2023 10:55:38 -0300 Subject: [PATCH 12/14] flake8 --- dandi/tests/test_dandiapi.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index a41d1ce01..ac11c1ccb 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -705,7 +705,7 @@ def test_rename_type_mismatch(text_dandiset: SampleDandiset, dest: str) -> None: def test_dandiset_has_data_standard(): with DandiAPIClient() as client: - dandiset = client.get_dandiset("000003", version_id='0.210812.1448') + dandiset = client.get_dandiset("000003", version_id="0.210812.1448") assert dandiset.has_data_standard("NWB") assert dandiset.has_data_standard("RRID:SCR_015242") assert not dandiset.has_data_standard("RRID:XXX_000000") @@ -714,7 +714,11 @@ def test_dandiset_has_data_standard(): def test_dandiset_has_data_standard_incorrect_arg(): with DandiAPIClient() as client: - dandiset = client.get_dandiset("000003", version_id='0.210812.1448') + dandiset = client.get_dandiset("000003", version_id="0.210812.1448") with pytest.raises(ValueError) as exc_info: dandiset.has_data_standard("NWC") - assert str(exc_info.value) == "'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one of the following values: NWB, BIDS" + assert ( + str(exc_info.value) + == "'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one of the " + "following values: NWB, BIDS " + ) From 49085002a1ee526a440e4a739c3cfec328b07ca7 Mon Sep 17 00:00:00 2001 From: bendichter Date: Mon, 13 Mar 2023 12:36:21 -0300 Subject: [PATCH 13/14] rmv space --- dandi/tests/test_dandiapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index ac11c1ccb..8d1c140b2 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -720,5 +720,5 @@ def test_dandiset_has_data_standard_incorrect_arg(): assert ( str(exc_info.value) == "'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one of the " - "following values: NWB, BIDS " + "following values: NWB, BIDS" ) From a994e543e3883d6f628e987cc165e05383b3ab4e Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Thu, 30 Nov 2023 08:57:21 -0500 Subject: [PATCH 14/14] Update dandi/dandiapi.py Co-authored-by: Yaroslav Halchenko --- dandi/dandiapi.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 36b07de7a..1e9fb2df7 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1247,7 +1247,12 @@ def has_data_standard(self, data_standard: str) -> bool: "'data_standard' must be an RRID (of form 'RRID:XXX_NNNNNNN`) or one " f"of the following values: {', '.join(DATA_STANDARD_MAP.keys())}" ) - assets_summary = self.get_raw_metadata()["assetsSummary"] + assets_summary = self.get_raw_metadata().get("assetsSummary") + if assets_summary is None: + warnings.warn( + f"The raw metadata of RemoteDandiset {self.identifier} does not contain 'assetsSummary'. " + f"Assuming that it does not contain {data_standard}.") + return False if "dataStandard" not in assets_summary: return False return any(x["identifier"] == rrid for x in assets_summary["dataStandard"])