From 9bdba70881cf14453ff13e1f3e0286d7d4caaad1 Mon Sep 17 00:00:00 2001 From: Travis Thurber Date: Fri, 26 Jul 2024 09:52:18 -0700 Subject: [PATCH] replace pkg_resources --- mosartwmpy/config/config.py | 10 ++-- mosartwmpy/download.py | 12 ++-- mosartwmpy/farmer_abm/farmer_abm.py | 37 ++++++------ ...est_calculate_water_constraints_by_farm.py | 60 +++++++++---------- mosartwmpy/tests/test_model.py | 12 ++-- mosartwmpy/utilities/download_data.py | 12 ++-- 6 files changed, 73 insertions(+), 70 deletions(-) diff --git a/mosartwmpy/config/config.py b/mosartwmpy/config/config.py index ab1a2e8..52bbe1e 100644 --- a/mosartwmpy/config/config.py +++ b/mosartwmpy/config/config.py @@ -1,19 +1,19 @@ -import pkg_resources +import importlib.resources from benedict import benedict from benedict.dicts import benedict as Benedict def get_config(config_file_path: str = None) -> Benedict: """Configuration object for the model, using the Benedict type. - + Args: config_file_path (string): path to the user defined configuration yaml file - + Returns: Benedict: A Benedict instance containing the merged configuration """ - config = benedict(pkg_resources.resource_filename('mosartwmpy', 'config_defaults.yaml'), format='yaml') + config = benedict(str(importlib.resources.files('mosartwmpy').joinpath('config_defaults.yaml')), format='yaml') if config_file_path is not None and config_file_path != '': config.merge(benedict(str(config_file_path), format='yaml'), overwrite=True) - + return config diff --git a/mosartwmpy/download.py b/mosartwmpy/download.py index 8aa663a..debfab2 100644 --- a/mosartwmpy/download.py +++ b/mosartwmpy/download.py @@ -1,9 +1,9 @@ from benedict import benedict from mosartwmpy.utilities.download_data import download_data from pathlib import Path -import pkg_resources +import importlib.resources -available_data = benedict.from_yaml(pkg_resources.resource_filename('mosartwmpy', 'data_manifest.yaml')) +available_data = benedict.from_yaml(str(importlib.resources.files('mosartwmpy').joinpath('data_manifest.yaml'))) data_list = [] data = [] @@ -27,9 +27,9 @@ {d}""") print(f""" - + 0) exit - + """) try: user_input = int(input(""" @@ -39,9 +39,9 @@ if not user_input or user_input == 0 or user_input > len(data): print(""" - + Exiting... - + """) exit() diff --git a/mosartwmpy/farmer_abm/farmer_abm.py b/mosartwmpy/farmer_abm/farmer_abm.py index a8a12c6..6b2d801 100644 --- a/mosartwmpy/farmer_abm/farmer_abm.py +++ b/mosartwmpy/farmer_abm/farmer_abm.py @@ -2,7 +2,7 @@ import numpy as np from os import mkdir import pandas as pd -from pyomo.environ import ConcreteModel, Constraint, maximize, NonNegativeReals, Objective, Set, Param, Set, Var +from pyomo.environ import ConcreteModel, Constraint, maximize, NonNegativeReals, Objective, Set, Param, Set, Var from pyomo.opt import SolverFactory from timeit import default_timer as timer import xarray as xr @@ -15,7 +15,7 @@ class FarmerABM: def __init__(self, model): self.model = model - self.config = model.config + self.config = model.config self.processed_years = [] # Get variables from the config. @@ -34,7 +34,7 @@ def __init__(self, model): self.nldas_id = get_config_variable_name(self, 'simulation.grid_output', 'nldas_id') self.reservoir_storage = get_config_variable_name(self, 'simulation.output', 'reservoir_storage') self.runoff_land = get_config_variable_name(self, 'simulation.output', 'runoff_land') - + def calc_demand(self): """Calculates water demand for each farmer using an agent-based model(ABM) and outputs into a parquet file. Requires input files: @@ -42,7 +42,7 @@ def calc_demand(self): * land_water_constraints_by_farm.parquet * crop_prices_by_nldas_id.parquet """ - + logging.info("\nRunning farmer ABM. ") t = timer() year = self.model.current_time.year @@ -87,7 +87,7 @@ def calc_demand(self): logging.info(f"Loaded positive mathematical programming (PMP) calibration files for year {year}.") # Number of crop and NLDAS ID combinations. - ids = range(len(crop_prices_by_nldas_id)) + ids = range(len(crop_prices_by_nldas_id)) # Number of farm agents / NLDAS IDs. farm_ids = range(len(pd.unique(crop_prices_by_nldas_id[self.config.get('water_management.demand.farmer_abm.crop_prices_by_nldas_id.variables.nldas_id')]))) crop_ids_by_farm = crop_prices_by_nldas_id.drop(columns='index').reset_index().groupby(by=self.config.get('water_management.demand.farmer_abm.crop_prices_by_nldas_id.variables.nldas_id'))['index'].apply(list) @@ -115,8 +115,8 @@ def calc_demand(self): # Construct functions. def obj_fun(fwm_s): - # .00001 is a scaling factor for computational purposes (doesn't influence optimization results). - # 0.5 is part of the positive mathematical formulation equation. + # .00001 is a scaling factor for computational purposes (doesn't influence optimization results). + # 0.5 is part of the positive mathematical formulation equation. # Both values will not vary between runs. return 0.00001*sum(sum((fwm_s.net_prices[i] * fwm_s.xs[i] - 0.5 * fwm_s.gammas[i] * fwm_s.xs[i] * fwm_s.xs[i]) for i in fwm_s.crop_ids_by_farm[f]) for f in fwm_s.farm_ids) fwm_s.obj_f = Objective(rule=obj_fun, sense=maximize) @@ -155,15 +155,15 @@ def water_constraint(fwm_s, ff): # Export results to parquet. results_pd = results_pd[[ - self.config.get('water_management.demand.farmer_abm.crop_prices_by_nldas_id.variables.nldas_id'), - self.config.get('water_management.demand.farmer_abm.crop_prices_by_nldas_id.variables.crop'), + self.config.get('water_management.demand.farmer_abm.crop_prices_by_nldas_id.variables.nldas_id'), + self.config.get('water_management.demand.farmer_abm.crop_prices_by_nldas_id.variables.crop'), self.config.get('water_management.demand.farmer_abm.crop_prices_by_nldas_id.variables.calculated_area') ]] # Create output directory if it doesn't already exist. - try: - mkdir(output_dir) - except OSError as error: + try: + mkdir(output_dir) + except OSError as error: logging.error(error) results_pd.to_parquet(f"{output_dir}/{self.config.get('simulation.name')}_farmer_abm_results_{str(year)}.parquet") @@ -198,7 +198,7 @@ def water_constraint(fwm_s, ff): self.processed_years.append(year) except Exception as e: logging.exception(str(e)) - + logging.info(f"Ran farmer ABM in {pretty_timer(timer() - t)}. This does not indicate success or failure. ") @@ -216,14 +216,16 @@ def calculate_water_constraints_by_farm(self, land_water_constraints_by_farm): reservoir_parameter_path = self.config.get('water_management.reservoirs.parameters.path') simulation_output_path = f"{self.config.get('simulation.output_path')}/{self.config.get('simulation.name')}/{self.config.get('simulation.name')}_{self.model.current_time.year-1}_*.nc" - # Map between grid cell ID and the cell that is dependent upon it (many to many). + # Map between grid cell ID and the cell that is dependent upon it (many to many). historic_storage_supply = pd.read_parquet(historic_storage_supply_path) # Relationships between grid cells and reservoirs they can consume from (many to many). dependency_database = pd.read_parquet(dependency_database_path) # Determines which grid cells the reservoirs are located in (one to one). - reservoir_parameters = xr.open_dataset(reservoir_parameter_path)[[self.reservoir_id, self.reservoir_grid_index]].to_dataframe() + reservoir_parameters_xr = xr.open_dataset(reservoir_parameter_path)[[self.reservoir_id, self.reservoir_grid_index]] + reservoir_parameters = reservoir_parameters_xr.to_dataframe() + reservoir_parameters_xr.close() # Get mosartwmpy output. simulation_output_xr = xr.open_mfdataset(simulation_output_path) @@ -231,6 +233,7 @@ def calculate_water_constraints_by_farm(self, land_water_constraints_by_farm): self.grid_cell_id, self.reservoir_storage, self.grid_cell_supply, self.runoff_land, self.nldas_id ]].mean('time').to_dataframe().reset_index() simulation_output[self.nldas_id] = simulation_output_xr[self.nldas_id].isel(time=0).to_dataframe().reset_index()[self.nldas_id].values + simulation_output_xr.close() # Merge the dependencies with the reservoir grid cells. dependency_database = dependency_database.merge(reservoir_parameters, how='left', on=self.reservoir_id).rename(columns={self.reservoir_grid_index: self.config.get('water_management.reservoirs.dependencies.variables.reservoir_cell_index')}) @@ -243,7 +246,7 @@ def calculate_water_constraints_by_farm(self, land_water_constraints_by_farm): ) # Merge in the mean supply and mean channel outflow from the simulation results per grid cell. - abm_data[[ + abm_data[[ self.grid_cell_supply, self.runoff_land ]] = abm_data[[self.dependent_cell_index]].merge(simulation_output[[ self.grid_cell_id, self.grid_cell_supply, self.runoff_land @@ -299,4 +302,4 @@ def calculate_water_constraints_by_farm(self, land_water_constraints_by_farm): water_constraints_by_farm = abm_data.reset_index()[WRM_SUPPLY_BIAS_CORRECTION].to_dict() logging.info(f"Converted units dataframe for year {self.model.current_time.year}") - return water_constraints_by_farm \ No newline at end of file + return water_constraints_by_farm diff --git a/mosartwmpy/tests/test_calculate_water_constraints_by_farm.py b/mosartwmpy/tests/test_calculate_water_constraints_by_farm.py index 565346b..231879c 100644 --- a/mosartwmpy/tests/test_calculate_water_constraints_by_farm.py +++ b/mosartwmpy/tests/test_calculate_water_constraints_by_farm.py @@ -1,7 +1,7 @@ import unittest import numpy as np import pandas as pd -import pkg_resources +import importlib.resources from mosartwmpy import Model from mosartwmpy.farmer_abm.farmer_abm import FarmerABM @@ -10,19 +10,19 @@ class CalculateWaterConstraintsByFarmTest(unittest.TestCase): - GRID_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/grid.zip') - OUTPUT_PATH = pkg_resources.resource_filename('mosartwmpy', 'tests') - CONFIG_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/test_config.yaml') - RUNOFF_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/runoff_1981_01_01.nc') - DEMAND_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/demand_1981_01_01.nc') - RESERVOIRS_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/reservoirs.nc') - DEPENDENCY_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/dependency_database.parquet') - MEAN_FLOW_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/mean_flow.parquet') - MEAN_DEMAND_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/mean_demand.parquet') - CONSTRAINTS_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/test_land_water_constraints_by_farm.parquet') - LIVE_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/test_land_water_constraints_by_farm_live.parquet') - BIAS_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/test_historic_storage_supply_bias.parquet') - CROP_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/test_crop_prices_by_nldas_id.parquet') + GRID_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'grid.zip')) + OUTPUT_PATH = str(importlib.resources.files('mosartwmpy').joinpath('tests')) + CONFIG_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'test_config.yaml')) + RUNOFF_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'runoff_1981_01_01.nc')) + DEMAND_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'demand_1981_01_01.nc')) + RESERVOIRS_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'reservoirs.nc')) + DEPENDENCY_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'dependency_database.parquet')) + MEAN_FLOW_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'mean_flow.parquet')) + MEAN_DEMAND_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'mean_demand.parquet')) + CONSTRAINTS_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'test_land_water_constraints_by_farm.parquet')) + LIVE_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'test_land_water_constraints_by_farm_live.parquet')) + BIAS_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'test_historic_storage_supply_bias.parquet')) + CROP_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'test_crop_prices_by_nldas_id.parquet')) def test_calculate_water_constraints_by_farm(self): model = Model() @@ -48,22 +48,22 @@ def test_calculate_water_constraints_by_farm(self): land_water_constraints_by_farm = pd.read_parquet(land_water_constraints_by_farm_path) water_constraints_by_farm = farmerABM.calculate_water_constraints_by_farm(land_water_constraints_by_farm) - expected = {0: 30.72285118452137, 1: 56.40536982827646, 2: 3.0684983145957405, 3: 37.66540204352916, 4: 83.92334320542051, 5: 16.471781747342227, - 6: 233.10350292402126, 7: 2552.1110474347847, 8: 39.77842459395363, 9: 1794.121981735524, 10: 222.93698006044025, 11: 5816.8735676312635, - 12: 384.71756436000385, 13: 33.75622675565492, 14: 8.992966440755907, 15: 13915.186859239666, 16: 5332.013002733427, 17: 9.440670507910413, - 18: 19.680088488121392, 19: 2.593803835847434, 20: 226.0076990680527, 21: 21.23900104164071, 22: 12.338676766223513, 23: 74.46348552407855, - 24: 654.7202788173175, 25: 326.74447842874287, 26: 49.27986148382904, 27: 15.594722020424333, 28: 2.042073646707058, 29: 10.87679754091625, - 30: 0.0, 31: 9.101107376292541, 32: 85.6742114585276, 33: 211.35930661507126, 34: 607.7330637330299, 35: 483.5908697772146, - 36: 1176.3042993778172, 37: 2615.216044397955, 38: 3299.4129275649575, 39: 3288.416218192872, 40: 5645.160759609735, 41: 3111.019269761427, - 42: 1694.3293926337408, 43: 1613.3569833770598, 44: 10813.511979632387, 45: 12215.247525518405, 46: 25394.201806792233, 47: 14588.669019674044, - 48: 369.8737720511687, 49: 122.81606333922728, 50: 8.323512859757498, 51: 3652.4126529340183, 52: 18174.9365444444, 53: 19144.127546798303, - 54: 1534.553615542034, 55: 220.74229489581606, 56: 277.4710702842115, 57: 0.09653429406007573, 58: 565.8425932097445, 59: 12.728978339437774, - 60: 0.772274352480606, 61: 1551.6497118991267, 62: 1392.7267419231105, 63: 7221.506155647856, 64: 828.13864024198, 65: 0.0, - 66: 0.6118340282182333, 67: 0.0, 68: 58.32028963190562, 69: 232.47165527754677, 70: 0.734200833460332, 71: 5.059971435408126, - 72: 3.4965916238134627, 73: 1431.8118059436317, 74: 1485.4959445184363, 75: 2.584332432101624, 76: 6.6368652768786855, 77: 6.409825493793453, - 78: 1.4802583101439315, 79: 42.93789870356224, 80: 478.64376563589775, 81: 320.83639169291024, 82: 115.03324054540802, 83: 983.5804008315356, - 84: 1203.645132678576, 85: 498.9473799328982, 86: 2571.661013189811, 87: 2306.8185854646376, 88: 257.65481107310165, 89: 1426.1978236790142, - 90: 1867.5493916929483, 91: 871.2380665376529, 92: 551.7301491571899, 93: 12.085666626862842, 94: 0.0, 95: 0.0, + expected = {0: 30.72285118452137, 1: 56.40536982827646, 2: 3.0684983145957405, 3: 37.66540204352916, 4: 83.92334320542051, 5: 16.471781747342227, + 6: 233.10350292402126, 7: 2552.1110474347847, 8: 39.77842459395363, 9: 1794.121981735524, 10: 222.93698006044025, 11: 5816.8735676312635, + 12: 384.71756436000385, 13: 33.75622675565492, 14: 8.992966440755907, 15: 13915.186859239666, 16: 5332.013002733427, 17: 9.440670507910413, + 18: 19.680088488121392, 19: 2.593803835847434, 20: 226.0076990680527, 21: 21.23900104164071, 22: 12.338676766223513, 23: 74.46348552407855, + 24: 654.7202788173175, 25: 326.74447842874287, 26: 49.27986148382904, 27: 15.594722020424333, 28: 2.042073646707058, 29: 10.87679754091625, + 30: 0.0, 31: 9.101107376292541, 32: 85.6742114585276, 33: 211.35930661507126, 34: 607.7330637330299, 35: 483.5908697772146, + 36: 1176.3042993778172, 37: 2615.216044397955, 38: 3299.4129275649575, 39: 3288.416218192872, 40: 5645.160759609735, 41: 3111.019269761427, + 42: 1694.3293926337408, 43: 1613.3569833770598, 44: 10813.511979632387, 45: 12215.247525518405, 46: 25394.201806792233, 47: 14588.669019674044, + 48: 369.8737720511687, 49: 122.81606333922728, 50: 8.323512859757498, 51: 3652.4126529340183, 52: 18174.9365444444, 53: 19144.127546798303, + 54: 1534.553615542034, 55: 220.74229489581606, 56: 277.4710702842115, 57: 0.09653429406007573, 58: 565.8425932097445, 59: 12.728978339437774, + 60: 0.772274352480606, 61: 1551.6497118991267, 62: 1392.7267419231105, 63: 7221.506155647856, 64: 828.13864024198, 65: 0.0, + 66: 0.6118340282182333, 67: 0.0, 68: 58.32028963190562, 69: 232.47165527754677, 70: 0.734200833460332, 71: 5.059971435408126, + 72: 3.4965916238134627, 73: 1431.8118059436317, 74: 1485.4959445184363, 75: 2.584332432101624, 76: 6.6368652768786855, 77: 6.409825493793453, + 78: 1.4802583101439315, 79: 42.93789870356224, 80: 478.64376563589775, 81: 320.83639169291024, 82: 115.03324054540802, 83: 983.5804008315356, + 84: 1203.645132678576, 85: 498.9473799328982, 86: 2571.661013189811, 87: 2306.8185854646376, 88: 257.65481107310165, 89: 1426.1978236790142, + 90: 1867.5493916929483, 91: 871.2380665376529, 92: 551.7301491571899, 93: 12.085666626862842, 94: 0.0, 95: 0.0, 96: 2.757034402743735, 97: 2.059543992483289, 98: 0.0, 99: 0.0} np.testing.assert_almost_equal( diff --git a/mosartwmpy/tests/test_model.py b/mosartwmpy/tests/test_model.py index 2e67627..3dbe14c 100644 --- a/mosartwmpy/tests/test_model.py +++ b/mosartwmpy/tests/test_model.py @@ -1,7 +1,7 @@ import numpy as np import unittest -import pkg_resources +import importlib.resources from mosartwmpy import Model from mosartwmpy.grid.grid import Grid @@ -11,11 +11,11 @@ class ModelTest(unittest.TestCase): """Test that the model initializes and runs with the default settings.""" # package data - GRID_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/grid.zip') - CONFIG_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/test_config.yaml') - RUNOFF_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/runoff_1981_01_01.nc') - DEMAND_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/demand_1981_01_01.nc') - RESERVOIRS_FILE = pkg_resources.resource_filename('mosartwmpy', 'tests/reservoirs.nc') + GRID_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'grid.zip')) + CONFIG_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'test_config.yaml')) + RUNOFF_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'runoff_1981_01_01.nc')) + DEMAND_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'demand_1981_01_01.nc')) + RESERVOIRS_FILE = str(importlib.resources.files('mosartwmpy').joinpath('tests', 'reservoirs.nc')) @classmethod def setUpClass(self): diff --git a/mosartwmpy/utilities/download_data.py b/mosartwmpy/utilities/download_data.py index 8b0a413..e4cbb6c 100644 --- a/mosartwmpy/utilities/download_data.py +++ b/mosartwmpy/utilities/download_data.py @@ -2,7 +2,7 @@ import logging import os from pathlib import Path -import pkg_resources +import importlib.resources import requests import sys from tqdm import tqdm @@ -11,12 +11,12 @@ from benedict import benedict -def download_data(dataset: str, destination: str = None, manifest: str = pkg_resources.resource_filename('mosartwmpy', 'data_manifest.yaml')) -> None: +def download_data(dataset: str, destination: str = None, manifest: str = str(importlib.resources.files('mosartwmpy').joinpath('data_manifest.yaml'))) -> None: """Convenience wrapper for the InstallSupplement class. - + Download and unpack example data supplement from Zenodo that matches the current installed distribution. - + Args: dataset (str): name of the dataset to download, as found in the data_manifest.yaml destination (str): full path to the directory in which to unpack the downloaded files; must be write enabled; defaults to the directory listed in the manifest @@ -24,10 +24,10 @@ def download_data(dataset: str, destination: str = None, manifest: str = pkg_res """ data_dictionary = benedict(manifest, format='yaml') - + if not data_dictionary.get(dataset, None): raise Exception(f'Dataset "{dataset}" not found in the manifest ({manifest}).') - + get = InstallSupplement( url=data_dictionary.get(f'{dataset}.url'), destination=destination if destination is not None else Path(data_dictionary.get(f'{dataset}.destination', './'))