From e80946adb9dc239be3ee2d335851ebce0330b3d7 Mon Sep 17 00:00:00 2001 From: kim Date: Tue, 7 Nov 2023 13:26:30 -0900 Subject: [PATCH] collects dataset concept ids by shortNames --- asf_search/CMR/__init__.py | 2 +- asf_search/CMR/datasets.py | 695 +++++++++++++++++------------------- asf_search/CMR/translate.py | 7 +- tests/Search/test_search.py | 41 +-- 4 files changed, 342 insertions(+), 403 deletions(-) diff --git a/asf_search/CMR/__init__.py b/asf_search/CMR/__init__.py index 9a9e9d10..905e8eb3 100644 --- a/asf_search/CMR/__init__.py +++ b/asf_search/CMR/__init__.py @@ -2,4 +2,4 @@ from .subquery import build_subqueries from .translate import translate_product, translate_opts, get_additional_fields from .field_map import field_map -from .datasets import platform_datasets +from .datasets import dataset_collections \ No newline at end of file diff --git a/asf_search/CMR/datasets.py b/asf_search/CMR/datasets.py index 342e48e3..b9bf61b6 100644 --- a/asf_search/CMR/datasets.py +++ b/asf_search/CMR/datasets.py @@ -1,370 +1,327 @@ -platform_datasets = { - "SENTINEL-1": [ - "C1214470488-ASF", - "C1327985661-ASF", - "C1214470533-ASF", - "C1214470576-ASF", - "C1327985645-ASF", - "C1214470496-ASF", - "C1214470532-ASF", - "C1214472977-ASF", - "C1214472336-ASF", - "C1266376001-ASF", - "C1214470732-ASF", - "C1327985660-ASF", - "C1327985741-ASF", - "C1327985578-ASF", - "C1327985650-ASF", - "C1327985617-ASF", - "C1327985579-ASF", - "C1327985619-ASF", - "C1214472994-ASF", - "C1214473170-ASF", - "C1327985646-ASF", - "C1327985740-ASF", - "C1327985739-ASF", - "C1214470561-ASF", - "C1214471521-ASF", - "C1214470682-ASF", - "C1327985647-ASF", - "C1214471197-ASF", - "C1214471960-ASF", - "C1214472978-ASF", - "C1214473165-ASF", - "C1327985697-ASF", - "C1327985651-ASF", - "C1327985644-ASF", - "C1327985571-ASF", - "C1327985674-ASF", - "C1205428742-ASF", - "C1216244348-ASF", - "C1212201032-ASF", - "C1216244589-ASF", - "C1212212560-ASF", - "C1205264459-ASF", - "C1216244594-ASF", - "C1216244593-ASF", - "C1208117434-ASF", - "C1212209035-ASF", - "C1212209226-ASF", - "C1216244585-ASF", - "C1216244592-ASF", - "C1208115009-ASF", - "C1216244595-ASF", - "C1212158327-ASF", - "C1215704763-ASF", - "C1212158318-ASF", - "C1216244590-ASF", - "C1216244601-ASF", - "C1216244600-ASF", - "C1212212493-ASF", - "C1212158326-ASF", - "C1212233976-ASF", - "C1216244591-ASF", - "C1216244587-ASF", - "C1216244598-ASF", - "C1216244586-ASF", - "C1212200781-ASF", - "C1212209075-ASF", - "C1216244597-ASF", - "C1216244596-ASF", - "C1216244588-ASF", - "C1216244599-ASF", - "C1234413245-ASFDEV", - "C1234413263-ASFDEV", - "C1234413229-ASFDEV", - "C1234413247-ASFDEV", - "C1234413237-ASFDEV", - "C1234413238-ASFDEV", - "C1234413248-ASFDEV", - "C1234413255-ASFDEV", - "C1234413236-ASFDEV", - "C1234413230-ASFDEV", - "C1234413232-ASFDEV", - "C1234413254-ASFDEV", - "C1234413256-ASFDEV", - "C1234413235-ASFDEV", - "C1234413253-ASFDEV", - "C1234413240-ASFDEV", - "C1234413234-ASFDEV", - "C1234413241-ASFDEV", - "C1234413252-ASFDEV", - "C1234413250-ASFDEV", - "C1234413259-ASFDEV", - "C1234413233-ASFDEV", - "C1234413243-ASFDEV", - "C1234413244-ASFDEV", - "C1234413251-ASFDEV", - "C1234413261-ASFDEV", - "C1234413262-ASFDEV", - "C1234413258-ASFDEV", - "C1244552887-ASFDEV", - "C1234413228-ASFDEV", - "C1234413231-ASFDEV", - "C1234413239-ASFDEV", - "C1234413242-ASFDEV", - "C1234413246-ASFDEV", - "C1234413249-ASFDEV", - "C1234413257-ASFDEV", - "C1234413260-ASFDEV", - "C1257175154-ASFDEV", - "C1244598379-ASFDEV", - "C1240784657-ASFDEV", - ], - "OPERA-S1": [ - "C2777443834-ASF", - "C2777436413-ASF", - "C2795135174-ASF", # RTC-STATIC - "C2795135668-ASF", # CSLC-STATIC - "C1258354200-ASF", # maybe extra? - "C1259982010-ASF", - "C1257995185-ASF", - "C1259976861-ASF", - "C1258354201-ASF", - "C1259981910-ASF", - "C1257995186-ASF", - "C1259974840-ASF", - "C1260726384-ASF", - "C1260721945-ASF", - "C1260721853-ASF", - "C1260726378-ASF", - - ], - "SLC-BURST": ["C2709161906-ASF", "C1257024016-ASF"], - "ALOS PALSAR": [ - "C1206487504-ASF", - "C1206485940-ASF", - "C1206487217-ASF", - "C1206156901-ASF", - "C1206485320-ASF", - "C1206485527-ASF", - "C2011599335-ASF", - "C1207181535-ASF", - "C1207710476-ASF", - "C1239927797-ASF", - "C1205261223-ASF", - "C1208013295-ASF", - "C1207019609-ASF", - "C1229740239-ASF", - "C1239611505-ASFDEV", - "C1238733834-ASFDEV", - ], - "ALOS AVNIR-2": [ - "C1808440897-ASF", - "C1234712303-ASF", - "C1233629671-ASF", - "C1234413224-ASFDEV", - ], - "SIR-C": [ - "C1661710578-ASF", - "C1661710581-ASF", - "C1661710583-ASF", - "C1661710586-ASF", - "C1661710588-ASF", - "C1661710590-ASF", - "C1661710593-ASF", - "C1661710596-ASF", - "C1661710597-ASF", - "C1661710600-ASF", - "C1661710603-ASF", - "C1661710604-ASF", - "C1226557813-ASF", - "C1226557814-ASF", - "C1226557819-ASF", - "C1226557809-ASF", - "C1226557808-ASF", - "C1226557810-ASF", - "C1226557811-ASF", - "C1226557812-ASF", - "C1226557815-ASF", - "C1226557816-ASF", - "C1226557817-ASF", - "C1226557818-ASF", - "C1234413270-ASFDEV", - "C1234413271-ASFDEV", - "C1234413264-ASFDEV", - "C1234413265-ASFDEV", - "C1234413266-ASFDEV", - "C1234413267-ASFDEV", - "C1234413268-ASFDEV", - "C1234413269-ASFDEV", - "C1234413272-ASFDEV", - "C1234413273-ASFDEV", - "C1234413274-ASFDEV", - "C1234413275-ASFDEV", - ], - "ARIA S1 GUNW": [ - "C1595422627-ASF", - "C1596065640-ASF", - "C1596065639-ASF", - "C1596065641-ASF", - "C1595765183-ASF", - "C1225776654-ASF", - "C1225776655-ASF", - "C1225776657-ASF", - "C1225776658-ASF", - "C1225776659-ASF", - ], - "SMAP": [ - "C1243122884-ASF", - "C1243124139-ASF", - "C1214473171-ASF", - "C1243149604-ASF", - "C1214473426-ASF", - "C1243119801-ASF", - "C1214473839-ASF", - "C1243133204-ASF", - "C1243197402-ASF", - "C1243215430-ASF", - "C1243124754-ASF", - "C1243141638-ASF", - "C1243162394-ASF", - "C1243168733-ASF", - "C1243168866-ASF", - "C1214473308-ASF", - "C1243253631-ASF", - "C1243133445-ASF", - "C1214473550-ASF", - "C1243197502-ASF", - "C1243126328-ASF", - "C1214474243-ASF", - "C1243216659-ASF", - "C1243129847-ASF", - "C1214473367-ASF", - "C1243268956-ASF", - "C1243144528-ASF", - "C1214473624-ASF", - "C1243228612-ASF", - "C1243136142-ASF", - "C1214474435-ASF", - "C1243255360-ASF", - "C1243140611-ASF", - "C1233103964-ASF", - "C1216074923-ASF", - "C1212243761-ASF", - "C1213091807-ASF", - "C1216074922-ASF", - "C1212243437-ASF", - "C1213096699-ASF", - "C1216074750-ASF", - "C1212249653-ASF", - "C1213101573-ASF", - "C1216074751-ASF", - "C1213136240-ASF", - "C1216074755-ASF", - "C1213136752-ASF", - "C1213136799-ASF", - "C1213136709-ASF", - "C1213136844-ASF", - "C1212249811-ASF", - "C1213125007-ASF", - "C1216074919-ASF", - "C1212196951-ASF", - "C1213115690-ASF", - "C1216074758-ASF", - "C1212243666-ASF", - "C1213115896-ASF", - "C1216074761-ASF", - "C1212250364-ASF", - "C1213134622-ASF", - "C1216074770-ASF", - "C1212246173-ASF", - "C1213125156-ASF", - "C1216074764-ASF", - "C1212249773-ASF", - "C1213134486-ASF", - "C1233101609-ASF", - ], - "UAVSAR": [ - "C1214353986-ASF", - "C1214336717-ASF", - "C1214336045-ASF", - "C1214335430-ASF", - "C1214335471-ASF", - "C1214335903-ASF", - "C1214336154-ASF", - "C1214336554-ASF", - "C1214353593-ASF", - "C1214353754-ASF", - "C1214353859-ASF", - "C1214337770-ASF", - "C1214354144-ASF", - "C1214354235-ASF", - "C1214343609-ASF", - "C1214354031-ASF", - "C1214408428-ASF", - "C1214419355-ASF", - "C1210487703-ASF", - "C1212030772-ASF", - "C1206116665-ASF", - "C1206132445-ASF", - "C1211962154-ASF", - "C1212001698-ASF", - "C1212005594-ASF", - "C1212019993-ASF", - "C1207638502-ASF", - "C1210025872-ASF", - "C1210485039-ASF", - "C1207188317-ASF", - "C1210546638-ASF", - "C1206122195-ASF", - "C1209970710-ASF", - "C1207038647-ASF", - "C1210599503-ASF", - "C1210599673-ASF", - ], - "RADARSAT-1": [ - "C1206897141-ASF", - "C1206936391-ASF", - "C1205181982-ASF", - "C1215670813-ASF", - "C1215709884-ASF", - "C1215709880-ASF", - "C1215714443-ASF", - "C1215775284-ASF", - "C1215614037-ASF", - ], - "ERS": [ - "C1210197768-ASF", - "C1211627521-ASF", - "C1208794942-ASF", - "C1209373626-ASF", - "C1207143701-ASF", - "C1205261222-ASF", - "C1205302527-ASF", - "C1207144966-ASF", - ], - "JERS-1": [ - "C1208662092-ASF", - "C1207933168-ASF", - "C1207175327-ASF", - "C1207177736-ASF", - ], - "AIRSAR": [ - "C1213921661-ASF", - "C1213921626-ASF", - "C1213928843-ASF", - "C1213927035-ASF", - "C179001730-ASF", - "C1213925022-ASF", - "C1213926419-ASF", - "C1213926777-ASF", - "C1213927939-ASF", - "C1213928209-ASF", - "C1205256880-ASF", - "C1208652494-ASF", - "C1000000306-ASF", - "C1208713702-ASF", - "C1208707768-ASF", - "C1208655639-ASF", - "C1208680681-ASF", - "C1208691361-ASF", - "C1208703384-ASF", - ], - "SEASAT": [ - "C1206500826-ASF", - "C1206500991-ASF", - "C1206752770-ASF", - "C1206144699-ASF", - ], +dataset_collections = { + "SENTINEL-1": { + "SENTINEL-1A_SLC": ["C1214470488-ASF", "C1205428742-ASF", "C1234413245-ASFDEV"], + "SENTINEL-1B_SLC": ["C1327985661-ASF", "C1216244348-ASF", "C1234413263-ASFDEV"], + "SENTINEL-1A_DP_GRD_HIGH": [ + "C1214470533-ASF", + "C1212201032-ASF", + "C1234413229-ASFDEV", + ], + "SENTINEL-1A_DP_META_GRD_HIGH": [ + "C1214470576-ASF", + "C1212209226-ASF", + "C1234413232-ASFDEV", + ], + "SENTINEL-1B_DP_GRD_HIGH": [ + "C1327985645-ASF", + "C1216244589-ASF", + "C1234413247-ASFDEV", + ], + "SENTINEL-1A_META_SLC": [ + "C1214470496-ASF", + "C1208117434-ASF", + "C1234413236-ASFDEV", + ], + "SENTINEL-1A_META_RAW": [ + "C1214470532-ASF", + "C1208115009-ASF", + "C1234413235-ASFDEV", + ], + "SENTINEL-1A_OCN": ["C1214472977-ASF", "C1212212560-ASF", "C1234413237-ASFDEV"], + "SENTINEL-1A_DP_META_GRD_MEDIUM": [ + "C1214472336-ASF", + "C1212212493-ASF", + "C1234413233-ASFDEV", + ], + "SENTINEL-1A_META_OCN": [ + "C1266376001-ASF", + "C1215704763-ASF", + "C1234413234-ASFDEV", + ], + "SENTINEL-1A_SP_META_GRD_HIGH": [ + "C1214470732-ASF", + "C1212158326-ASF", + "C1234413243-ASFDEV", + ], + "SENTINEL-1B_DP_GRD_MEDIUM": [ + "C1327985660-ASF", + "C1216244594-ASF", + "C1234413248-ASFDEV", + ], + "SENTINEL-1B_DP_META_GRD_HIGH": [ + "C1327985741-ASF", + "C1216244601-ASF", + "C1234413250-ASFDEV", + ], + "SENTINEL-1B_DP_META_GRD_MEDIUM": [ + "C1327985578-ASF", + "C1216244591-ASF", + "C1234413251-ASFDEV", + ], + "SENTINEL-1B_META_RAW": [ + "C1327985650-ASF", + "C1216244595-ASF", + "C1234413253-ASFDEV", + ], + "SENTINEL-1B_META_SLC": [ + "C1327985617-ASF", + "C1216244585-ASF", + "C1234413254-ASFDEV", + ], + "SENTINEL-1B_OCN": ["C1327985579-ASF", "C1216244593-ASF", "C1234413255-ASFDEV"], + "SENTINEL-1B_SP_META_GRD_HIGH": [ + "C1327985619-ASF", + "C1216244587-ASF", + "C1234413261-ASFDEV", + ], + "SENTINEL-1A_SP_GRD_MEDIUM": [ + "C1214472994-ASF", + "C1212158318-ASF", + "C1234413241-ASFDEV", + ], + "SENTINEL-1A_SP_META_GRD_MEDIUM": [ + "C1214473170-ASF", + "C1212233976-ASF", + "C1234413244-ASFDEV", + ], + "SENTINEL-1B_META_OCN": [ + "C1327985646-ASF", + "C1216244590-ASF", + "C1234413252-ASFDEV", + ], + "SENTINEL-1B_SP_GRD_MEDIUM": [ + "C1327985740-ASF", + "C1216244600-ASF", + "C1234413259-ASFDEV", + ], + "SENTINEL-1B_SP_META_GRD_MEDIUM": [ + "C1327985739-ASF", + "C1216244598-ASF", + "C1234413262-ASFDEV", + ], + "SENTINEL-1A_RAW": ["C1214470561-ASF", "C1205264459-ASF", "C1234413238-ASFDEV"], + "SENTINEL-1A_DP_GRD_MEDIUM": [ + "C1214471521-ASF", + "C1212209035-ASF", + "C1234413230-ASFDEV", + ], + "SENTINEL-1A_SP_GRD_HIGH": [ + "C1214470682-ASF", + "C1212158327-ASF", + "C1234413240-ASFDEV", + ], + "SENTINEL-1B_RAW": ["C1327985647-ASF", "C1216244592-ASF", "C1234413256-ASFDEV"], + "SENTINEL-1A_DP_GRD_FULL": [ + "C1214471197-ASF", + "C1212200781-ASF", + "C1234413228-ASFDEV", + ], + "SENTINEL-1A_DP_META_GRD_FULL": [ + "C1214471960-ASF", + "C1212209075-ASF", + "C1234413231-ASFDEV", + ], + "SENTINEL-1A_SP_GRD_FULL": ["C1214472978-ASF", "C1234413239-ASFDEV"], + "SENTINEL-1A_SP_META_GRD_FULL": ["C1214473165-ASF", "C1234413242-ASFDEV"], + "SENTINEL-1B_DP_GRD_FULL": [ + "C1327985697-ASF", + "C1216244597-ASF", + "C1234413246-ASFDEV", + ], + "SENTINEL-1B_DP_META_GRD_FULL": [ + "C1327985651-ASF", + "C1216244596-ASF", + "C1234413249-ASFDEV", + ], + "SENTINEL-1B_SP_GRD_FULL": [ + "C1327985644-ASF", + "C1216244588-ASF", + "C1234413257-ASFDEV", + ], + "SENTINEL-1B_SP_GRD_HIGH": [ + "C1327985571-ASF", + "C1216244586-ASF", + "C1234413258-ASFDEV", + ], + "SENTINEL-1B_SP_META_GRD_FULL": [ + "C1327985674-ASF", + "C1216244599-ASF", + "C1234413260-ASFDEV", + ], + "S1_Bursts": ["C1244552887-ASFDEV"], + "SENTINEL-1_BURSTS_DEV10": ["C1257175154-ASFDEV"], + "Sentinel-1_Burst_Map": ["C1244598379-ASFDEV"], + "Various Browse Images": ["C1240784657-ASFDEV"], + }, + "OPERA-S1": { + "OPERA_L2_CSLC-S1_V1": ["C2777443834-ASF", "C1259976861-ASF"], + "OPERA_L2_RTC-S1_V1": ["C2777436413-ASF", "C1259974840-ASF"], + "OPERA_L2_CSLC-S1-STATIC_PROVISIONAL_V0": ["C1258354200-ASF"], + "OPERA_L2_CSLC-S1-STATIC_V1": ["C1259982010-ASF"], + "OPERA_L2_CSLC-S1_PROVISIONAL_V0": ["C1257995185-ASF"], + "OPERA_L2_RTC-S1-STATIC_PROVISIONAL_V0": ["C1258354201-ASF"], + "OPERA_L2_RTC-S1-STATIC_V1": ["C1259981910-ASF"], + "OPERA_L2_RTC-S1_PROVISIONAL_V0": ["C1257995186-ASF"], + "OPERA_L2_CSLC-S1-STATIC_CALVAL_V1": ["C1260726384-ASF"], + "OPERA_L2_CSLC-S1_CALVAL_V1": ["C1260721945-ASF"], + "OPERA_L2_RTC-S1_CALVAL_V1": ["C1260721853-ASF"], + "OPERA_L2_RTC-S1-STATIC_CALVAL_V1": ["C1260726378-ASF"], + }, + "SLC-BURST": {"SENTINEL-1_BURSTS": ["C2709161906-ASF", "C1257024016-ASF"]}, + "ALOS PALSAR": { + "ALOS_PSR_RTC_HIGH": ["C1206487504-ASF", "C1207181535-ASF"], + "ALOS_PSR_L1.5": ["C1206485940-ASF", "C1205261223-ASF"], + "ALOS_PSR_RTC_LOW": ["C1206487217-ASF", "C1208013295-ASF"], + "ALOS_PSR_KMZ": ["C1206156901-ASF", "C1207019609-ASF"], + "ALOS_PSR_L1.0": ["C1206485320-ASF"], + "ALOS_PSR_L1.1": ["C1206485527-ASF", "C1207710476-ASF", "C1239611505-ASFDEV"], + "ALOS_PSR_L2.2": ["C2011599335-ASF", "C1239927797-ASF", "C1238733834-ASFDEV"], + "ALOS_PALSAR_INSAR_METADATA": ["C1229740239-ASF"], + }, + "ALOS AVNIR-2": { + "ALOS_AVNIR_OBS_ORI": [ + "C1808440897-ASF", + "C1233629671-ASF", + "C1234413224-ASFDEV", + ], + "ALOS_AVNIR_OBS_ORI_BROWSE": ["C1234712303-ASF"], + }, + "SIR-C": { + "STS-59_BROWSE_GRD": [ + "C1661710578-ASF", + "C1226557819-ASF", + "C1234413264-ASFDEV", + ], + "STS-59_BROWSE_SLC": [ + "C1661710581-ASF", + "C1226557809-ASF", + "C1234413265-ASFDEV", + ], + "STS-59_GRD": ["C1661710583-ASF", "C1226557808-ASF", "C1234413266-ASFDEV"], + "STS-59_META_GRD": ["C1661710586-ASF", "C1226557810-ASF", "C1234413267-ASFDEV"], + "STS-59_META_SLC": ["C1661710588-ASF", "C1226557811-ASF", "C1234413268-ASFDEV"], + "STS-59_SLC": ["C1661710590-ASF", "C1226557812-ASF", "C1234413269-ASFDEV"], + "STS-68_BROWSE_GRD": [ + "C1661710593-ASF", + "C1226557813-ASF", + "C1234413270-ASFDEV", + ], + "STS-68_BROWSE_SLC": [ + "C1661710596-ASF", + "C1226557814-ASF", + "C1234413271-ASFDEV", + ], + "STS-68_GRD": ["C1661710597-ASF", "C1226557815-ASF", "C1234413272-ASFDEV"], + "STS-68_META_GRD": ["C1661710600-ASF", "C1226557816-ASF", "C1234413273-ASFDEV"], + "STS-68_META_SLC": ["C1661710603-ASF", "C1226557817-ASF", "C1234413274-ASFDEV"], + "STS-68_SLC": ["C1661710604-ASF", "C1226557818-ASF", "C1234413275-ASFDEV"], + }, + "ARIA S1 GUNW": { + "SENTINEL-1_INTERFEROGRAMS": ["C1595422627-ASF", "C1225776654-ASF"], + "SENTINEL-1_INTERFEROGRAMS_AMPLITUDE": ["C1596065640-ASF", "C1225776655-ASF"], + "SENTINEL-1_INTERFEROGRAMS_COHERENCE": ["C1596065639-ASF", "C1225776657-ASF"], + "SENTINEL-1_INTERFEROGRAMS_CONNECTED_COMPONENTS": [ + "C1596065641-ASF", + "C1225776658-ASF", + ], + "SENTINEL-1_INTERFEROGRAMS_UNWRAPPED_PHASE": [ + "C1595765183-ASF", + "C1225776659-ASF", + ], + }, + "SMAP": { + "SPL1A_RO_METADATA_003": ["C1243122884-ASF", "C1233103964-ASF"], + "SPL1A_RO_QA_003": ["C1243124139-ASF", "C1216074923-ASF"], + "SPL1A_001": ["C1214473171-ASF", "C1212243761-ASF"], + "SPL1A_002": ["C1243149604-ASF", "C1213091807-ASF"], + "SPL1A_METADATA_001": ["C1214473426-ASF", "C1212243437-ASF"], + "SPL1A_METADATA_002": ["C1243119801-ASF", "C1213096699-ASF"], + "SPL1A_QA_001": ["C1214473839-ASF", "C1212249653-ASF"], + "SPL1A_QA_002": ["C1243133204-ASF", "C1213101573-ASF"], + "SPL1A_RO_001": ["C1243197402-ASF"], + "SPL1A_RO_002": ["C1243215430-ASF", "C1213136240-ASF"], + "SPL1A_RO_003": ["C1243124754-ASF", "C1216074755-ASF"], + "SPL1A_RO_METADATA_001": ["C1243141638-ASF", "C1213136752-ASF"], + "SPL1A_RO_METADATA_002": ["C1243162394-ASF", "C1213136799-ASF"], + "SPL1A_RO_QA_001": ["C1243168733-ASF", "C1213136709-ASF"], + "SPL1A_RO_QA_002": ["C1243168866-ASF", "C1213136844-ASF"], + "SPL1B_SO_LoRes_001": ["C1214473308-ASF", "C1212249811-ASF"], + "SPL1B_SO_LoRes_002": ["C1243253631-ASF", "C1213125007-ASF"], + "SPL1B_SO_LoRes_003": ["C1243133445-ASF", "C1216074919-ASF"], + "SPL1B_SO_LoRes_METADATA_001": ["C1214473550-ASF", "C1212196951-ASF"], + "SPL1B_SO_LoRes_METADATA_002": ["C1243197502-ASF", "C1213115690-ASF"], + "SPL1B_SO_LoRes_METADATA_003": ["C1243126328-ASF", "C1216074758-ASF"], + "SPL1B_SO_LoRes_QA_001": ["C1214474243-ASF", "C1212243666-ASF"], + "SPL1B_SO_LoRes_QA_002": ["C1243216659-ASF", "C1213115896-ASF"], + "SPL1B_SO_LoRes_QA_003": ["C1243129847-ASF", "C1216074761-ASF"], + "SPL1C_S0_HiRes_001": ["C1214473367-ASF", "C1212250364-ASF"], + "SPL1C_S0_HiRes_002": ["C1243268956-ASF", "C1213134622-ASF"], + "SPL1C_S0_HiRes_003": ["C1243144528-ASF", "C1216074770-ASF"], + "SPL1C_S0_HiRes_METADATA_001": ["C1214473624-ASF", "C1212246173-ASF"], + "SPL1C_S0_HiRes_METADATA_002": ["C1243228612-ASF", "C1213125156-ASF"], + "SPL1C_S0_HiRes_METADATA_003": ["C1243136142-ASF", "C1216074764-ASF"], + "SPL1C_S0_HiRes_QA_001": ["C1214474435-ASF", "C1212249773-ASF"], + "SPL1C_S0_HiRes_QA_002": ["C1243255360-ASF", "C1213134486-ASF"], + "SPL1C_S0_HiRes_QA_003": ["C1243140611-ASF", "C1233101609-ASF"], + "SPL1A_003": ["C1216074922-ASF"], + "SPL1A_METADATA_003": ["C1216074750-ASF"], + "SPL1A_QA_003": ["C1216074751-ASF"], + }, + "UAVSAR": { + "UAVSAR_POL_META": ["C1214353986-ASF", "C1210487703-ASF"], + "UAVSAR_INSAR_META": ["C1214336717-ASF", "C1212030772-ASF"], + "UAVSAR_INSAR_INT": ["C1214336045-ASF", "C1212001698-ASF"], + "UAVSAR_INSAR_AMP": ["C1214335430-ASF", "C1206116665-ASF"], + "UAVSAR_INSAR_AMP_GRD": ["C1214335471-ASF", "C1206132445-ASF"], + "UAVSAR_INSAR_DEM": ["C1214335903-ASF", "C1211962154-ASF"], + "UAVSAR_INSAR_INT_GRD": ["C1214336154-ASF", "C1212005594-ASF"], + "UAVSAR_INSAR_KMZ": ["C1214336554-ASF", "C1212019993-ASF"], + "UAVSAR_POL_DEM": ["C1214353593-ASF", "C1207638502-ASF"], + "UAVSAR_POL_INC": ["C1214353754-ASF", "C1210025872-ASF"], + "UAVSAR_POL_KMZ": ["C1214353859-ASF", "C1210485039-ASF"], + "UAVSAR_POL_ML_CMPLX_GRD": ["C1214337770-ASF", "C1207188317-ASF"], + "UAVSAR_POL_ML_CMPLX_GRD_3X3": ["C1214354144-ASF", "C1210546638-ASF"], + "UAVSAR_POL_ML_CMPLX_GRD_5X5": ["C1214354235-ASF", "C1206122195-ASF"], + "UAVSAR_POL_ML_CMPLX_SLANT": ["C1214343609-ASF", "C1209970710-ASF"], + "UAVSAR_POL_PAULI": ["C1214354031-ASF", "C1207038647-ASF"], + "UAVSAR_POL_SLOPE": ["C1214408428-ASF", "C1210599503-ASF"], + "UAVSAR_POL_STOKES": ["C1214419355-ASF", "C1210599673-ASF"], + }, + "RADARSAT-1": { + "RSAT-1_L0": ["C1206897141-ASF"], + "RSAT-1_L1": ["C1206936391-ASF", "C1205181982-ASF"], + "RSAT-1_POLAR_YEAR_ANTARCTICA_L1": ["C1215670813-ASF"], + "RSAT-1_POLAR_YEAR_GREENLAND_L0": ["C1215709884-ASF"], + "RSAT-1_POLAR_YEAR_GREENLAND_L1": ["C1215709880-ASF"], + "RSAT-1_POLAR_YEAR_KAMCHATKA_L1": ["C1215714443-ASF"], + "RSAT-1_POLAR_YEAR_SEA_ICE_MIN_MAX_L1": ["C1215775284-ASF"], + "RSAT-1_POLAR_YEAR_TOOLIK_L1": ["C1215614037-ASF"], + }, + "ERS": { + "ERS-1_L0": ["C1210197768-ASF", "C1205261222-ASF"], + "ERS-1_L1": ["C1211627521-ASF", "C1205302527-ASF"], + "ERS-2_L0": ["C1208794942-ASF", "C1207143701-ASF"], + "ERS-2_L1": ["C1209373626-ASF", "C1207144966-ASF"], + }, + "JERS-1": { + "JERS-1_L0": ["C1208662092-ASF", "C1207175327-ASF"], + "JERS-1_L1": ["C1207933168-ASF", "C1207177736-ASF"], + }, + "AIRSAR": { + "AIRSAR_POL_3FP": ["C1213921661-ASF", "C1205256880-ASF"], + "AIRSAR_INT_JPG": ["C1213921626-ASF", "C1000000306-ASF"], + "AIRSAR_POL_SYN_3FP": ["C1213928843-ASF", "C1208713702-ASF"], + "AIRSAR_TOP_C-DEM_STOKES": ["C1213927035-ASF", "C1208707768-ASF"], + "AIRSAR_TOP_DEM": ["C179001730-ASF", "C1208655639-ASF"], + "AIRSAR_TOP_DEM_C": ["C1213925022-ASF", "C1208680681-ASF"], + "AIRSAR_TOP_DEM_L": ["C1213926419-ASF", "C1208691361-ASF"], + "AIRSAR_TOP_DEM_P": ["C1213926777-ASF", "C1208703384-ASF"], + "AIRSAR_TOP_L-STOKES": ["C1213927939-ASF"], + "AIRSAR_TOP_P-STOKES": ["C1213928209-ASF"], + "AIRSAR_INT": ["C1208652494-ASF"], + }, + "SEASAT": { + "SEASAT_SAR_L1_TIFF": ["C1206500826-ASF", "C1206752770-ASF"], + "SEASAT_SAR_L1_HDF5": ["C1206500991-ASF", "C1206144699-ASF"], + }, } diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index 703bbfda..348a33dc 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -7,7 +7,7 @@ from shapely.geometry import Polygon from shapely.geometry.base import BaseGeometry from .field_map import field_map -from .datasets import platform_datasets +from .datasets import dataset_collections import logging @@ -53,8 +53,9 @@ def translate_opts(opts: ASFSearchOptions) -> list: dict_opts['collections'] = [] for dataset in dict_opts['datasets']: - if dataset_collections := platform_datasets.get(dataset): - dict_opts['collections'].extend(dataset_collections) + if collections_by_short_name := dataset_collections.get(dataset): + for shortName, concept_ids in collections_by_short_name.items(): + dict_opts['collections'].extend(concept_ids) else: raise ValueError(f'Could not find dataset named "{dataset}" provided for datasets keyword.') diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index 5b9f3989..d22a9f54 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -6,13 +6,9 @@ from asf_search.exceptions import ASFSearchError from asf_search.search import search from asf_search.ASFSearchResults import ASFSearchResults -from asf_search.CMR import platform_datasets - -import requests -from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type - +from asf_search.CMR import dataset_collections from pytest import raises - +import requests import requests_mock def run_test_ASFSearchResults(search_resp): @@ -78,39 +74,24 @@ def custom_matcher(request: requests.Request): with raises(ASFSearchError): results.raise_if_incomplete() -@retry( - reraise=True, - wait=wait_fixed(3), - stop=stop_after_attempt(3), - retry=retry_if_exception_type((requests.ReadTimeout, requests.HTTPError)) -) def test_datasets_search(): datasets = ['SENTINEL-1', 'RADARSAT-1', 'UAVSAR'] - should_raise_error = len([dataset for dataset in datasets if not platform_datasets.get(dataset)]) > 0 + should_raise_error = len([dataset for dataset in datasets if not dataset_collections.get(dataset)]) > 0 if should_raise_error: with raises(ValueError): search(datasets=datasets, maxResults=1) - - # get collection concept-ids from shortName cmr query else: - valid_collections = [] + valid_shortnames = [] for dataset in datasets: - valid_collections.extend(platform_datasets.get(dataset)) + valid_shortnames.extend([shortName for shortName in dataset_collections.get(dataset).keys()]) response = search(datasets=datasets, maxResults=250) - # Granules don't keep track of their collection concept-ids, - # but they do have the collection names we can use to find them! - shortNames = '&'.join(list(set([f"shortName[]={get(product.umm, 'CollectionReference', 'ShortName')}" for product in response]))) - - r = requests.get(f"https://cmr.earthdata.nasa.gov/search/collections.umm_json?{shortNames}&provider=ASF") - r.raise_for_status() - - items = r.json()['items'] - concept_ids = [item['meta']['concept-id'] for item in items] - - # check that results are limited to the expected datasets - for concept_id in concept_ids: - assert concept_id in valid_collections + # Get collection shortName of all granules + shortNames = list(set([get(product.umm, 'CollectionReference', 'ShortName') for product in response])) + + # and check that results are limited to the expected datasets by their shortname + for shortName in shortNames: + assert shortName in valid_shortnames \ No newline at end of file