From 0f969e302766c257b87c005d6e5f1877f1cba7e3 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Mon, 24 Oct 2022 14:54:04 +0200 Subject: [PATCH 1/6] refcator to_netcdf to get config_file instead of dict --- PyStemmusScope/save.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/PyStemmusScope/save.py b/PyStemmusScope/save.py index 848936bb..66ada9bb 100644 --- a/PyStemmusScope/save.py +++ b/PyStemmusScope/save.py @@ -24,7 +24,7 @@ import numpy as np import pandas as pd import xarray as xr -from PyStemmusScope import forcing_io +from PyStemmusScope import forcing_io, config_io from . import variable_conversion as vc @@ -250,18 +250,21 @@ def _update_dataset_attrs_dims(dataset: xr.Dataset, forcing_dict: Dict) -> xr.Da return dataset -def to_netcdf(config: Dict, cf_filename: str) -> str: +def to_netcdf(config_file: str, cf_filename: str) -> str: """Save csv files generated by Stemmus_Scope model to a netcdf file using information provided by ALMA conventions. Args: - config(Dict): PyStemmusScope configuration dictionary. + config_file(str): Path to the config file. cf_filename(str): Path to a csv file for ALMA conventions. Returns: str: path to a csv file under the output directory. """ + # read config file + config = config_io.read_config(config_file) + # list of required forcing variables, Alma_short_name: forcing_io_name, # model_name var_names = { "RH": "rh", # RH From d6d695a9338636beb1326eceaeac49c10ef9f1c1 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Mon, 24 Oct 2022 14:54:23 +0200 Subject: [PATCH 2/6] update the notebooks --- notebooks/run_model_in_notebook.ipynb | 4 ++-- notebooks/run_model_in_notebook_dev.ipynb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/notebooks/run_model_in_notebook.ipynb b/notebooks/run_model_in_notebook.ipynb index 9a9e5b6c..8dc3614f 100644 --- a/notebooks/run_model_in_notebook.ipynb +++ b/notebooks/run_model_in_notebook.ipynb @@ -217,7 +217,7 @@ "source": [ "# save output in netcdf format\n", "required_netcdf_variables = \"./required_netcdf_variables.csv\"\n", - "nc_file_name = save.to_netcdf(model.config, required_netcdf_variables)\n", + "nc_file_name = save.to_netcdf(config_path, required_netcdf_variables)\n", "print(nc_file_name)" ] }, @@ -266,7 +266,7 @@ " print(result)\n", " \n", " # save results in a netcdf file\n", - " nc_file_name = save.to_netcdf(model.config, required_netcdf_variables)\n", + " nc_file_name = save.to_netcdf(config_path, required_netcdf_variables)\n", " print(nc_file_name)\n", " " ] diff --git a/notebooks/run_model_in_notebook_dev.ipynb b/notebooks/run_model_in_notebook_dev.ipynb index 7a5e8193..952d89cc 100644 --- a/notebooks/run_model_in_notebook_dev.ipynb +++ b/notebooks/run_model_in_notebook_dev.ipynb @@ -235,7 +235,7 @@ ], "source": [ "required_netcdf_variables = path_to_model / \"utils\" / \"csv_to_nc\" / \"required_netcdf_variables.csv\"\n", - "nc_file_name = save.to_netcdf(model.config, required_netcdf_variables)\n", + "nc_file_name = save.to_netcdf(config_path, required_netcdf_variables)\n", "print(nc_file_name)" ] }, From cce98d4e1deb602453d4b83b29072d0b217a2dad Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Mon, 24 Oct 2022 15:05:59 +0200 Subject: [PATCH 3/6] fix tests --- tests/test_save.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/test_save.py b/tests/test_save.py index 3948296f..363d4040 100644 --- a/tests/test_save.py +++ b/tests/test_save.py @@ -45,16 +45,16 @@ def model_with_setup(self, tmp_path): with patch("time.strftime") as mocked_time: mocked_time.return_value = "2022-08-01-1200" - _ = model.setup( + config_path = model.setup( WorkDir = str(tmp_path), ForcingFileName = "dummy_forcing_file.nc", NumberOfTimeSteps = "3", # less than forcing temporal range ) - return model + return model, config_path def test_save_to_netcdf(self, cf_convention, model_with_setup): - model = model_with_setup - saved_nc_file = save.to_netcdf(model.config, cf_convention) + model, config_path = model_with_setup + saved_nc_file = save.to_netcdf(config_path, cf_convention) expected_nc_file = ( "tests/test_data/directories/output/dummy-2022-08-01-1200/dummy-2022-08-01-1200_STEMMUS_SCOPE.nc" @@ -110,16 +110,16 @@ def model_with_setup(self, tmp_path): with patch("time.strftime") as mocked_time: mocked_time.return_value = "2022-08-01-1200" - _ = model.setup( + config_path = model.setup( WorkDir = str(tmp_path), ForcingFileName = "dummy_forcing_file.nc", NumberOfTimeSteps = "NA", # use temporal range of forcing data ) - return model + return model, config_path @pytest.fixture(name="_make_csv_file") def fixture_make_csv_file(self, model_with_setup): - model = model_with_setup + model, _ = model_with_setup data = [ "simulation_number,year,DoY,Netlong", ",,,W m-2", @@ -134,8 +134,8 @@ def fixture_make_csv_file(self, model_with_setup): write_csv(data, csv_file) def test_save_to_netcdf(self, cf_convention, _make_csv_file, model_with_setup): - model = model_with_setup - saved_nc_file = save.to_netcdf(model.config, cf_convention) + model, config_path = model_with_setup + saved_nc_file = save.to_netcdf(config_path, cf_convention) expected_nc_file = ( "tests/test_data/directories/output/dummy-2022-08-01-1200/dummy-2022-08-01-1200_STEMMUS_SCOPE.nc" @@ -193,16 +193,16 @@ def model_with_setup(self, tmp_path): with patch("time.strftime") as mocked_time: mocked_time.return_value = "2022-08-01-1200" - _ = model.setup( + config_path = model.setup( WorkDir = str(tmp_path), ForcingFileName = "dummy_forcing_file.nc", NumberOfTimeSteps = "5", ) - return model + return model, config_path @pytest.fixture(name="_make_csv_file") def fixture_make_csv_file(self, model_with_setup): - model = model_with_setup + model, _ = model_with_setup data = [ "1,2,3,5", "1,1,1,2", @@ -218,8 +218,8 @@ def fixture_make_csv_file(self, model_with_setup): write_csv(data, csv_file) def test_save_to_netcdf(self, cf_convention, _make_csv_file, model_with_setup): - model = model_with_setup - saved_nc_file = save.to_netcdf(model.config, cf_convention) + model, config_path = model_with_setup + saved_nc_file = save.to_netcdf(config_path, cf_convention) expected_nc_file = ( "tests/test_data/directories/output/dummy-2022-08-01-1200/dummy-2022-08-01-1200_STEMMUS_SCOPE.nc" @@ -282,16 +282,16 @@ def model_with_setup(self, tmp_path): with patch("time.strftime") as mocked_time: mocked_time.return_value = "2022-08-01-1200" - _ = model.setup( + config_path = model.setup( WorkDir = str(tmp_path), ForcingFileName = "dummy_forcing_file.nc", NumberOfTimeSteps = "NA", ) - return model + return model, config_path @pytest.fixture(name="_make_csv_file") def fixture_make_csv_file(self, model_with_setup): - model = model_with_setup + model, _ = model_with_setup data = [ "1,2,3,5", "1,1,1,2", @@ -318,8 +318,8 @@ def fixture_make_csv_file(self, model_with_setup): write_csv(data, csv_file) def test_save_to_netcdf(self, cf_convention, _make_csv_file, model_with_setup): - model = model_with_setup - saved_nc_file = save.to_netcdf(model.config, cf_convention) + model, config_path = model_with_setup + saved_nc_file = save.to_netcdf(config_path, cf_convention) expected_nc_file = ( "tests/test_data/directories/output/dummy-2022-08-01-1200/dummy-2022-08-01-1200_STEMMUS_SCOPE.nc" From 4bfce6a7e4a329aa5fca71ae5801b511e04a3c4a Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Mon, 24 Oct 2022 15:06:29 +0200 Subject: [PATCH 4/6] fix pylint too-many-locals --- PyStemmusScope/save.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/PyStemmusScope/save.py b/PyStemmusScope/save.py index 66ada9bb..b5941014 100644 --- a/PyStemmusScope/save.py +++ b/PyStemmusScope/save.py @@ -277,16 +277,13 @@ def to_netcdf(config_file: str, cf_filename: str) -> str: "Precip": "precip_conv", # Pre } - # Number of time steps from configuration file - time_steps = config["NumberOfTimeSteps"] - # read forcing file into a dict forcing_dict = forcing_io.read_forcing_data( Path(config["ForcingPath"]) / config["ForcingFileName"] ) # get time info - time = _shorten_data_array(forcing_dict["time"], time_steps) + time = _shorten_data_array(forcing_dict["time"], config["NumberOfTimeSteps"]) # read convention file conventions = pd.read_csv(cf_filename) @@ -300,7 +297,7 @@ def to_netcdf(config_file: str, cf_filename: str) -> str: if alma_name in var_names: # select data data_array = _select_forcing_variables(forcing_dict, var_names[alma_name], alma_name) - data_array = _shorten_data_array(data_array, time_steps) + data_array = _shorten_data_array(data_array, config["NumberOfTimeSteps"]) # create data array elif alma_name in {"SoilTemp", "SoilMoist"}: From 3aef498512457011142d003191cbd38749616c66 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Mon, 24 Oct 2022 15:17:34 +0200 Subject: [PATCH 5/6] fix pylint order of imports --- PyStemmusScope/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyStemmusScope/save.py b/PyStemmusScope/save.py index b5941014..8602204e 100644 --- a/PyStemmusScope/save.py +++ b/PyStemmusScope/save.py @@ -24,7 +24,7 @@ import numpy as np import pandas as pd import xarray as xr -from PyStemmusScope import forcing_io, config_io +from PyStemmusScope import config_io, forcing_io from . import variable_conversion as vc From b637b16afef91b7dca24095d2eb29a86e2fbd554 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Mon, 24 Oct 2022 15:37:05 +0200 Subject: [PATCH 6/6] run isort on save module --- PyStemmusScope/save.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PyStemmusScope/save.py b/PyStemmusScope/save.py index 8602204e..33610b14 100644 --- a/PyStemmusScope/save.py +++ b/PyStemmusScope/save.py @@ -24,7 +24,8 @@ import numpy as np import pandas as pd import xarray as xr -from PyStemmusScope import config_io, forcing_io +from PyStemmusScope import config_io +from PyStemmusScope import forcing_io from . import variable_conversion as vc