Skip to content

Commit

Permalink
Merge branch 'main' of github.com:ClimateImpactLab/pyCIAM into featur…
Browse files Browse the repository at this point in the history 
…e/update-for-ice-sheet
  • Loading branch information
@bolliger32
bolliger32 committed Apr 3, 2024
2 parents f965ba7 + a150234 commit 8780c3a
Show file tree
Hide file tree
Showing 14 changed files with 540 additions and 410 deletions.
9 changes: 4 additions & 5 deletions .github/workflows/python-publish.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@ jobs:
deploy:

runs-on: ubuntu-latest

permissions:
id-token: write
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: Install dependencies
Expand All @@ -30,5 +31,3 @@ jobs:
- name: Publish distribution 📦 to PyPI
if: startsWith(github.ref, 'refs/tags')
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.PYPI_API_TOKEN }}
10 changes: 8 additions & 2 deletions HISTORY.rst
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
History
=======

v1.1.3 (unreleased)
-------------------
unreleased
----------
* Update GitHub actions versions

v1.2.0
------
* Point `data-acquisition.ipynb` to updated Zenodo deposit that fixes the dtype of `subsets` variable in `diaz2016_inputs_raw.zarr.zip` to be bool rather than int8
* Variable name bugfix in `data-acquisition.ipynb`
* Add netcdf versions of SLIIDERS and the pyCIAM results to `upload-zenodo.ipynb`
* Update results in Zenodo record to use SLIIDERS v1.2

v1.1.2
------
Expand Down
136 changes: 64 additions & 72 deletions notebooks/models/create-surge-lookup-tables.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,23 @@
{
"cell_type": "code",
"execution_count": 1,
"id": "6b183aa3-366a-4f5f-bb95-5ef4bf48d438",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "cd4b2928-65a0-4733-9c71-fbefa85590be",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import sys\n",
Expand All @@ -41,16 +56,26 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 3,
"id": "afd03880-8e46-4d81-867d-bfb6505ea788",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/srv/conda/envs/notebook/lib/python3.9/site-packages/dask_gateway/client.py:21: FutureWarning: format_bytes is deprecated and will be removed in a future release. Please use dask.utils.format_bytes instead.\n",
" from distributed.utils import LoopRunner, format_bytes\n"
"/srv/conda/envs/notebook/lib/python3.10/site-packages/google/cloud/storage/transfer_manager.py:30: UserWarning: The module `transfer_manager` is a preview feature. Functionality and API may change. This warning will be removed in a future release.\n",
" warnings.warn(\n",
"/home/jovyan/git-repos/pyCIAM-public/notebooks/models/../shared.py:3: UserWarning: Shapely 2.0 is installed, but because PyGEOS is also installed, GeoPandas will still use PyGEOS by default for now. To force to use and test Shapely 2.0, you have to set the environment variable USE_PYGEOS=0. You can do this before starting the Python process, or in your code before importing geopandas:\n",
"\n",
"import os\n",
"os.environ['USE_PYGEOS'] = '0'\n",
"import geopandas\n",
"\n",
"In a future release, GeoPandas will switch to using Shapely by default. If you are using PyGEOS directly (calling PyGEOS functions on geometries from GeoPandas), this will then stop working and you are encouraged to migrate from PyGEOS to Shapely 2.0 (https://shapely.readthedocs.io/en/latest/migration_pygeos.html).\n",
" import geopandas as gpd\n"
]
}
],
Expand All @@ -75,7 +100,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"id": "47979c48-7fff-49b4-b445-ea6bf746fa37",
"metadata": {
"tags": [
Expand All @@ -86,18 +111,20 @@
"source": [
"# When running on larger/scalable dask cluster, may wish to specify number of workers\n",
"# Default is LocalCluster which will use the number of CPUs available on local machine\n",
"# N_WORKERS_MIN = 7\n",
"# N_WORKERS_MAX = 700\n",
"N_WORKERS_MIN = 7\n",
"N_WORKERS_MAX = 700\n",
"SEG_CHUNKSIZE = 5\n",
"\n",
"PARAMS = pd.read_json(PATH_PARAMS)[\"values\"]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 5,
"id": "6104bced",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"DMF_I = getattr(damage_funcs, PARAMS.dmf + \"_i\")\n",
Expand All @@ -106,70 +133,31 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"id": "dbb04b4a-14b8-4403-ad33-88bfe71bd8fc",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
" <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-f992a2c6-c6d1-11ed-862e-665816eadc72</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
"\n",
" <tr>\n",
" \n",
" <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
" <td style=\"text-align: left;\"><strong>Cluster type:</strong> dask_gateway.GatewayCluster</td>\n",
" \n",
" </tr>\n",
"\n",
" \n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status\" target=\"_blank\">/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" \n",
"\n",
" </table>\n",
"\n",
" \n",
" <details>\n",
" <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
" <div style='background-color: #f2f2f2; display: inline-block; padding: 10px; border: 1px solid #999999;'>\n",
" <h3>GatewayCluster</h3>\n",
" <ul>\n",
" <li><b>Name: </b>daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454\n",
" <li><b>Dashboard: </b><a href='/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status' target='_blank'>/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status</a>\n",
" </ul>\n",
"</div>\n",
"\n",
" </details>\n",
" \n",
"\n",
" </div>\n",
"</div>"
],
"application/vnd.jupyter.widget-view+json": {
"model_id": "fcadb94377b4430d9c2964ac9c808c2a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"<Client: 'tls://10.4.102.2:8786' processes=0 threads=0, memory=0 B>"
"VBox(children=(HTML(value='<h2>GatewayCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n<style scoped>\\n …"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
"output_type": "display_data"
}
],
"source": [
"client = start_dask_cluster(\n",
" # n_workers_min=N_WORKERS_MIN,\n",
" # n_workers_max=N_WORKERS_MAX,\n",
")\n",
"client"
"client, cluster = start_dask_cluster()\n",
"cluster.adapt(minimum=N_WORKERS_MIN, maximum=N_WORKERS_MAX)\n",
"cluster"
]
},
{
Expand All @@ -182,7 +170,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 7,
"id": "1e823775-0d12-4e52-93d1-7e3586913e65",
"metadata": {
"tags": []
Expand All @@ -205,7 +193,7 @@
" start_year=PARAMS.model_start,\n",
" slr_0_years=PARAMS.slr_0_year,\n",
" client=client,\n",
" # client_kwargs={\"batch_size\": N_WORKERS_MAX},\n",
" client_kwargs={\"batch_size\": N_WORKERS_MAX},\n",
" force_overwrite=True,\n",
" seg_chunksize=SEG_CHUNKSIZE,\n",
" mc_dim=\"quantile\",\n",
Expand All @@ -223,9 +211,11 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 11,
"id": "f941a509-7035-4105-8b79-88a97e759737",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# ensure completion and close cluster\n",
Expand All @@ -237,23 +227,25 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 12,
"id": "15563792-1ba0-435a-9981-e2b15eaa8dcd",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"(None, None)"
]
},
"execution_count": 10,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.cluster.close(), client.close()"
"cluster.close(), client.close()"
]
}
],
Expand All @@ -273,7 +265,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.10"
"version": "3.10.10"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
Expand Down
Loading

0 comments on commit 8780c3a

Please sign in to comment.