From 4da3cc18bad76c6b7d2288b883fa57586f84625f Mon Sep 17 00:00:00 2001 From: crvernon Date: Wed, 20 Jan 2021 17:32:56 -0500 Subject: [PATCH 1/4] add install supplement code for external data retrieval --- mosartwmpy/__init__.py | 3 +- mosartwmpy/external/install_supplement.py | 124 ++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 mosartwmpy/external/install_supplement.py diff --git a/mosartwmpy/__init__.py b/mosartwmpy/__init__.py index 7040c63..b01dc15 100644 --- a/mosartwmpy/__init__.py +++ b/mosartwmpy/__init__.py @@ -1 +1,2 @@ -from .model import Model \ No newline at end of file +from mosartwmpy.external.install_supplement import install_supplement +from mosartwmpy import Model diff --git a/mosartwmpy/external/install_supplement.py b/mosartwmpy/external/install_supplement.py new file mode 100644 index 0000000..0876dcc --- /dev/null +++ b/mosartwmpy/external/install_supplement.py @@ -0,0 +1,124 @@ +import os +import io +import requests +import zipfile +import logging +import sys + +from pkg_resources import get_distribution + + +def install_supplement(download_directory): + """Convenience wrapper for the InstallSupplement class. + + Download and unpack example data supplement from Zenodo that matches the current installed + distribution. + + :param download_directory: Full path to the directory you wish to install + the example data to. Must be write-enabled + for the user. + + """ + + get = InstallSupplement(example_data_directory=download_directory) + get.fetch_zenodo() + + +class InstallSupplement: + """Download and unpack example data supplement from Zenodo that matches the current installed + distribution. + + :param example_data_directory: Full path to the directory you wish to install + the example data to. Must be write-enabled + for the user. + + """ + + # URL for DOI minted example data hosted on Zenodo matching the version of release + # TODO: this dictionary should really be brought in from a config file within the package + # TODO: replace current test link with a real data link + DATA_VERSION_URLS = {'0.1.0': 'https://zenodo.org/record/3856417/files/test.zip?download=1'} + + def __init__(self, example_data_directory, model_name='mosart'): + + self.initialize_logger() + + # full path to the Xanthos root directory where the example dir will be stored + self.example_data_directory = self.valid_directory(example_data_directory) + + self.model_name = model_name + + def initialize_logger(self): + """Initialize logger to stdout.""" + + # initialize logger + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + # logger console handler + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) + logger.addHandler(console_handler) + + @staticmethod + def close_logger(): + """Shutdown logger.""" + + # Remove logging handlers + logger = logging.getLogger() + + for handler in logger.handlers[:]: + handler.close() + logger.removeHandler(handler) + + logging.shutdown() + + def valid_directory(self, directory): + """Ensure the provided directory exists.""" + + if os.path.isdir(directory): + return directory + else: + msg = f"The write directory provided by the user does not exist: {directory}" + logging.exception(msg) + self.close_logger() + raise NotADirectoryError(msg) + + def fetch_zenodo(self): + """Download and unpack the Zenodo example data supplement for the + current distribution.""" + + # get the current version of the package is installed + current_version = get_distribution(self.model_name).version + + try: + data_link = InstallSupplement.DATA_VERSION_URLS[current_version] + + except KeyError: + msg = f"Link to data missing for current version: {current_version}. Please contact admin." + logging.exception(msg) + self.close_logger() + raise + + # retrieve content from URL + try: + logging.info(f"Downloading example data for version {current_version} from {data_link}") + r = requests.get(data_link) + + with zipfile.ZipFile(io.BytesIO(r.content)) as zipped: + + # extract each file in the zipped dir to the project + for f in zipped.namelist(): + logging.info("Unzipped: {}".format(os.path.join(self.example_data_directory, f))) + zipped.extract(f, self.example_data_directory) + + logging.info("Download and install complete.") + + self.close_logger() + + except requests.exceptions.MissingSchema: + msg = f"URL for data incorrect for current version: {current_version}. Please contact admin." + logging.exception(msg) + self.close_logger() + raise From 459b65527307cb519ca15ae54b237eae81953e5e Mon Sep 17 00:00:00 2001 From: crvernon Date: Wed, 20 Jan 2021 17:52:21 -0500 Subject: [PATCH 2/4] update install protocol since repo name is not the same as the package --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc4d4b6..ce479a5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + python setup.py install - name: Test and generate coverage report run: | From a72d5edbcb0c1a5088a5e1e0d2a62ab51707de32 Mon Sep 17 00:00:00 2001 From: travis Date: Fri, 12 Feb 2021 14:43:58 -0800 Subject: [PATCH 3/4] adjust download script --- mosartwmpy/__init__.py | 3 +- mosartwmpy/data_manifest.yaml | 13 ++++ mosartwmpy/model.py | 9 ++- mosartwmpy/output/output.py | 2 +- .../download_data.py} | 60 +++++++------------ setup.py | 2 +- 6 files changed, 46 insertions(+), 43 deletions(-) create mode 100644 mosartwmpy/data_manifest.yaml rename mosartwmpy/{external/install_supplement.py => utilities/download_data.py} (58%) diff --git a/mosartwmpy/__init__.py b/mosartwmpy/__init__.py index b01dc15..7040c63 100644 --- a/mosartwmpy/__init__.py +++ b/mosartwmpy/__init__.py @@ -1,2 +1 @@ -from mosartwmpy.external.install_supplement import install_supplement -from mosartwmpy import Model +from .model import Model \ No newline at end of file diff --git a/mosartwmpy/data_manifest.yaml b/mosartwmpy/data_manifest.yaml new file mode 100644 index 0000000..2d74bdb --- /dev/null +++ b/mosartwmpy/data_manifest.yaml @@ -0,0 +1,13 @@ +# listing of publically downloadable data related to mosartwmpy + +sample_input: + description: Sample input dataset that can be used for testing and development; covers 1980 - 1985. + url: https://zenodo.org/record/4537907/files/mosartwmpy_sample_input_data_1980_1985.zip?download=1 + destination: ./ + +validation: + description: Result datasets that can be used for validating the model; includes results with and without WM; covers 1981-1982. + url: TODO + destination: ./validation + +# TODO add other weather files, demand files, etc as they become ready \ No newline at end of file diff --git a/mosartwmpy/model.py b/mosartwmpy/model.py index 228faee..b19214c 100644 --- a/mosartwmpy/model.py +++ b/mosartwmpy/model.py @@ -26,6 +26,7 @@ from mosartwmpy.reservoirs.reservoirs import reservoir_release from mosartwmpy.state.state import State from mosartwmpy.update.update import update +from mosartwmpy.utilities.download_data import download_data from mosartwmpy.utilities.pretty_timer import pretty_timer from mosartwmpy.utilities.inherit_docs import inherit_docs @@ -69,8 +70,8 @@ def initialize(self, config_file_path: str) -> None: self.parameters = Parameters() # sanitize the run name self.name = sanitize_filename(self.config.get('simulation.name')).replace(" ", "_") - # setup logging and output directory - Path(f'./output/{self.name}').mkdir(parents=True, exist_ok=True) + # setup logging and output directories + Path(f'./output/{self.name}/restart_files').mkdir(parents=True, exist_ok=True) logging.basicConfig( filename=f'./output/{self.name}/mosartwmpy.log', level=self.config.get('simulation.log_level', 'INFO'), @@ -195,6 +196,10 @@ def finalize(self) -> None: # simulation is over so free memory, write data, etc return + def download_data(self, *args, **kwargs) -> None: + """Downloads data related to the model.""" + download_data(*args, **kwargs) + def get_component_name(self) -> str: # TODO include version/hash info? return f'mosartwmpy ({self.git_hash})' diff --git a/mosartwmpy/output/output.py b/mosartwmpy/output/output.py index 86dd6e3..61a788e 100644 --- a/mosartwmpy/output/output.py +++ b/mosartwmpy/output/output.py @@ -157,5 +157,5 @@ def write_restart(self): logging.info('Writing restart file.') x = self.state.to_dataframe().to_xarray() - filename = f'./output/{self.name}/{self.name}_restart_{self.current_time.year}_{self.current_time.strftime("%m")}_{self.current_time.strftime("%d")}.nc' + filename = f'./output/{self.name}/restart_files/{self.name}_restart_{self.current_time.year}_{self.current_time.strftime("%m")}_{self.current_time.strftime("%d")}.nc' x.to_netcdf(filename) diff --git a/mosartwmpy/external/install_supplement.py b/mosartwmpy/utilities/download_data.py similarity index 58% rename from mosartwmpy/external/install_supplement.py rename to mosartwmpy/utilities/download_data.py index 0876dcc..1c6997d 100644 --- a/mosartwmpy/external/install_supplement.py +++ b/mosartwmpy/utilities/download_data.py @@ -5,22 +5,28 @@ import logging import sys +from benedict import benedict from pkg_resources import get_distribution -def install_supplement(download_directory): +def download_data(dataset: str, destination: str = None, manifest: str = './mosartwmpy/data_manifest.yaml') -> None: """Convenience wrapper for the InstallSupplement class. - + Download and unpack example data supplement from Zenodo that matches the current installed distribution. - - :param download_directory: Full path to the directory you wish to install - the example data to. Must be write-enabled - for the user. - + + Args: + dataset (str): name of the dataset to download, as found in the data_manifest.yaml + destination (str): full path to the directory in which to unpack the downloaded files; must be write enabled; defaults to the directory listed in the manifest + manifest (str): full path to the manifest yaml file describing the available downloads; defaults to the bundled data_manifest.yaml """ - get = InstallSupplement(example_data_directory=download_directory) + data_dictionary = benedict(manifest, format='yaml') + + if not data_dictionary.get(dataset, None): + raise Exception(f'Dataset "{dataset}" not found in the manifest ({manifest}).') + + get = InstallSupplement(url = data_dictionary.get(f'{dataset}.url'), destination = destination if destination is not None else data_dictionary.get(f'{dataset}.destination', './')) get.fetch_zenodo() @@ -34,19 +40,11 @@ class InstallSupplement: """ - # URL for DOI minted example data hosted on Zenodo matching the version of release - # TODO: this dictionary should really be brought in from a config file within the package - # TODO: replace current test link with a real data link - DATA_VERSION_URLS = {'0.1.0': 'https://zenodo.org/record/3856417/files/test.zip?download=1'} - - def __init__(self, example_data_directory, model_name='mosart'): + def __init__(self, url, destination): self.initialize_logger() - - # full path to the Xanthos root directory where the example dir will be stored - self.example_data_directory = self.valid_directory(example_data_directory) - - self.model_name = model_name + self.destination = self.valid_directory(destination) + self.url = url def initialize_logger(self): """Initialize logger to stdout.""" @@ -80,7 +78,7 @@ def valid_directory(self, directory): if os.path.isdir(directory): return directory else: - msg = f"The write directory provided by the user does not exist: {directory}" + msg = f"The write directory provided by the user does not exist: {directory}" logging.exception(msg) self.close_logger() raise NotADirectoryError(msg) @@ -89,36 +87,24 @@ def fetch_zenodo(self): """Download and unpack the Zenodo example data supplement for the current distribution.""" - # get the current version of the package is installed - current_version = get_distribution(self.model_name).version - - try: - data_link = InstallSupplement.DATA_VERSION_URLS[current_version] - - except KeyError: - msg = f"Link to data missing for current version: {current_version}. Please contact admin." - logging.exception(msg) - self.close_logger() - raise - # retrieve content from URL try: - logging.info(f"Downloading example data for version {current_version} from {data_link}") - r = requests.get(data_link) + logging.info(f"Downloading example data from {self.url}") + r = requests.get(self.url) with zipfile.ZipFile(io.BytesIO(r.content)) as zipped: # extract each file in the zipped dir to the project for f in zipped.namelist(): - logging.info("Unzipped: {}".format(os.path.join(self.example_data_directory, f))) - zipped.extract(f, self.example_data_directory) + logging.info("Unzipped: {}".format(os.path.join(self.destination, f))) + zipped.extract(f, self.destination) logging.info("Download and install complete.") self.close_logger() except requests.exceptions.MissingSchema: - msg = f"URL for data incorrect for current version: {current_version}. Please contact admin." + msg = f"Unable to download data from {self.url}" logging.exception(msg) self.close_logger() raise diff --git a/setup.py b/setup.py index 93d3486..d860ce9 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def get_requirements(): setup( name='mosartwmpy', - version='0.1.0', + version='0.0.1', packages=find_packages(), url='https://github.com/IMMM-SFA/mosartwmpy', license='BSD2', From e47acfd51739642079b7eb24c0025f5a7ad0f77a Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 15 Feb 2021 08:22:16 -0800 Subject: [PATCH 4/4] rebase and add download utility --- download.py | 53 +++++++++++++++++++++++++++ mosartwmpy/data_manifest.yaml | 2 +- mosartwmpy/utilities/download_data.py | 1 - 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 download.py diff --git a/download.py b/download.py new file mode 100644 index 0000000..ef915c8 --- /dev/null +++ b/download.py @@ -0,0 +1,53 @@ +import enum +import os + +from benedict import benedict + +from mosartwmpy.utilities.download_data import download_data + +available_data = benedict.from_yaml('./mosartwmpy/data_manifest.yaml') + +data_list = [] +data = [] + +for i, name in enumerate(available_data.keys()): + data_list.append(name) + data.append(f""" + {i + 1}) {name} - {available_data.get(f'{name}.description')}""") + +# clear the terminal +print(chr(27) + "[2J") + +print(f""" + 🎶 Welcome to the mosartwmpy download utility! 🎵 + + Please select the data you wish to download by typing the number: +""") + +for d in data: + print(f""" + {d}""") + +print(f""" + + 0) exit + +""") +try: + user_input = int(input(""" + Please select a number and press enter: """)) +except: + pass + +if not user_input or user_input == 0 or user_input > len(data): + print(""" + + Exiting... + + """) + +else: + print("") + print("") + download_data(data_list[user_input - 1]) + diff --git a/mosartwmpy/data_manifest.yaml b/mosartwmpy/data_manifest.yaml index 2d74bdb..1eade6e 100644 --- a/mosartwmpy/data_manifest.yaml +++ b/mosartwmpy/data_manifest.yaml @@ -7,7 +7,7 @@ sample_input: validation: description: Result datasets that can be used for validating the model; includes results with and without WM; covers 1981-1982. - url: TODO + url: https://zenodo.org/record/4539693/files/mosartwmpy_validation.zip?download=1 destination: ./validation # TODO add other weather files, demand files, etc as they become ready \ No newline at end of file diff --git a/mosartwmpy/utilities/download_data.py b/mosartwmpy/utilities/download_data.py index 1c6997d..bbea51d 100644 --- a/mosartwmpy/utilities/download_data.py +++ b/mosartwmpy/utilities/download_data.py @@ -6,7 +6,6 @@ import sys from benedict import benedict -from pkg_resources import get_distribution def download_data(dataset: str, destination: str = None, manifest: str = './mosartwmpy/data_manifest.yaml') -> None: