Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add netcdf versions of SLIIDERS and pyCIAM results to zenodo #8

Merged
merged 2 commits into from
Mar 4, 2024
Merged

Add netcdf versions of SLIIDERS and pyCIAM results to zenodo #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
afdf3f7
add netcdfs to upload
bolliger32 Mar 4, 2024
bda96c5
update HISTORY
bolliger32 Mar 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion HISTORY.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
History
=======

v1.1.3 (unreleased)
v1.2 (unreleased)
-------------------
* Point `data-acquisition.ipynb` to updated Zenodo deposit that fixes the dtype of `subsets` variable in `diaz2016_inputs_raw.zarr.zip` to be bool rather than int8
* Variable name bugfix in `data-acquisition.ipynb`
* Add netcdf versions of SLIIDERS and the pyCIAM results to `upload-zenodo.ipynb`

v1.1.2
------
Expand Down
100 changes: 66 additions & 34 deletions notebooks/post-processing/zenodo-upload.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,17 @@
{
"cell_type": "code",
"execution_count": 1,
"id": "42ac0087-fe1b-4088-919e-ce007e6dea8c",
"metadata": {},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "706cee66-6557-402d-ae97-679ed202a9fe",
"metadata": {
"tags": []
Expand All @@ -16,7 +27,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 3,
"id": "29a54dac-abdc-4a95-967f-2289fc1c9ecb",
"metadata": {
"tags": []
Expand All @@ -28,18 +39,33 @@
"from pathlib import Path\n",
"from shutil import make_archive\n",
"from tempfile import TemporaryDirectory\n",
"from pathlib import Path\n",
"\n",
"import dask.config\n",
"import requests\n",
"import shared\n",
"import xarray as xr\n",
"from sliiders import settings as sset\n",
"from sliiders.io import open_zarr\n",
"from zarr import ZipStore"
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"id": "f804bd08-6edb-43f6-8bbd-7a2b98281c29",
"metadata": {},
"outputs": [],
"source": [
"PATH_SLIIDERS_NC = sset.PATH_SLIIDERS.parent / (sset.PATH_SLIIDERS.stem + \".nc\")\n",
"PATH_OUTPUTS_NC = shared.PATH_OUTPUTS.parent / (shared.PATH_OUTPUTS.stem + \".nc\")\n",
"\n",
"shared.save_dataset(shared.open_zarr(sset.PATH_SLIIDERS).load(), PATH_SLIIDERS_NC)\n",
"shared.save_dataset(shared.open_zarr(sset.PATH_OUTPUTS).load(), PATH_OUTPUTS_NC)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d37ca26f-091d-4cb6-be02-385464e0b687",
"metadata": {
"tags": []
Expand All @@ -48,10 +74,10 @@
{
"data": {
"text/plain": [
"<dask.config.set at 0x112a25b70>"
"<dask.config.set at 0x7cfe44438890>"
]
},
"execution_count": 3,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -70,7 +96,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 21,
"id": "55a3e849-a554-49fa-882c-846acf76f3b8",
"metadata": {
"tags": [
Expand All @@ -79,19 +105,19 @@
},
"outputs": [],
"source": [
"ACCESS_TOKEN = environ[\"ACCESS_TOKEN\"]\n",
"VERSION = \"1.1.0\"\n",
"ACCESS_TOKEN = \"Q5z5IQ1m5Z9l1QS7ZYeV78IS5bqPmhzcFVo0KSNLoh2p39HRMPgFoJsCyQt5\"\n",
"VERSION = \"1.2.0\"\n",
"TITLES = {\n",
" \"SLIIDERS\": \"SLIIDERS: Sea Level Impacts Input Dataset by Elevation, Region, and Scenario\",\n",
" \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n",
" # \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n",
"}\n",
"PYCIAM_CODE_PATH=Path(\"pyCIAM-1.1.2.zip\")\n",
"SLIIDERS_CODE_PATH=Path(\"sliiders-1.1.1.zip\")"
"PYCIAM_CODE_PATH = Path(\"pyCIAM-1.1.2.zip\")\n",
"SLIIDERS_CODE_PATH = Path(\"/tmp/sliiders-1.2.zip\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 22,
"id": "8725b84c-8a7f-4d88-96ec-849e963ef8ce",
"metadata": {},
"outputs": [],
Expand All @@ -101,7 +127,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 23,
"id": "956c40ee-e3e8-4257-bd0d-2c938546604d",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -136,7 +162,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 24,
"id": "b5fcb2b8-3fb4-4789-bae0-18f7a66b7c51",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -206,15 +232,15 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 25,
"id": "55188020-c02f-4fa6-bf4c-cf1cd87e1754",
"metadata": {},
"outputs": [],
"source": [
"# Files and paths\n",
"ORIGINAL_PATHS = {\n",
" \"SLIIDERS\": {\n",
" \"products\": [shared.PATH_SLIIDERS],\n",
" \"products\": [sset.PATH_SLIIDERS, PATH_SLIIDERS_NC],\n",
" \"inputs\": [\n",
" sset.PATH_GEOG_GTSM_SNAPPED,\n",
" sset.PATH_GEOG_GTSM_STATIONS_TOTHIN,\n",
Expand All @@ -224,6 +250,7 @@
" \"pyCIAM\": {\n",
" \"products\": [\n",
" shared.PATH_OUTPUTS,\n",
" PATH_OUTPUTS_NC,\n",
" shared.PATH_DIAZ_RES,\n",
" shared.PATH_MOVEFACTOR_DATA,\n",
" ],\n",
Expand All @@ -240,7 +267,7 @@
"if PYCIAM_CODE_PATH is not None:\n",
" ORIGINAL_PATHS[\"pyCIAM\"][\"source\"] = [PYCIAM_CODE_PATH]\n",
"if SLIIDERS_CODE_PATH is not None:\n",
" ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH"
" ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH]"
]
},
{
Expand All @@ -255,7 +282,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 40,
"id": "50c10e4c-b3eb-4714-8d32-486ac878d041",
"metadata": {},
"outputs": [],
Expand All @@ -264,7 +291,7 @@
" dep = EXISTING_DEPOSITS[name]\n",
" # create new deposit\n",
" deposition_id = dep[\"id\"]\n",
" if deposition_id == int(dep[\"links\"][\"latest\"].split(\"/\")[-1]):\n",
" if \"latest_draft\" not in dep[\"links\"]:\n",
" url = f\"{Z_URL}/{deposition_id}/actions/newversion\"\n",
" r = requests.post(url, params=PARAMS)\n",
" if r.status_code not in [200, 201]:\n",
Expand Down Expand Up @@ -390,7 +417,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 41,
"id": "868b1d78-5eb4-4fa6-bcfc-0dbd8c88d39b",
"metadata": {},
"outputs": [],
Expand All @@ -408,23 +435,28 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 44,
"id": "ff8703bb-c630-418e-9f2f-c62b116ee7e8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pyCIAM\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/raw/CIAM_2016/diaz2016_inputs_raw.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/slr/ar5/ar5-msl-rel-2005-quantiles.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/int/exposure/ypk/finalized/ypk_2000_2100_20221122.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg_adm.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/pyCIAM_outputs.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/diaz2016_outputs.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/suboptimal_capital_by_movefactor.zarr\n"
"SLIIDERS\n"
]
},
{
"ename": "TypeError",
"evalue": "string indices must be integers, not 'str'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[44], line 6\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(name)\n\u001b[1;32m 4\u001b[0m this_dep \u001b[38;5;241m=\u001b[39m draft_deps[name]\n\u001b[0;32m----> 6\u001b[0m uploads[name] \u001b[38;5;241m=\u001b[39m \u001b[43mupload_file_list\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43mthis_dep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkind\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs/\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moverwrite\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[1;32m 8\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 9\u001b[0m uploads[name] \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m upload_file_list(\n\u001b[1;32m 10\u001b[0m this_dep, kind[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts\u001b[39m\u001b[38;5;124m\"\u001b[39m], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts/\u001b[39m\u001b[38;5;124m\"\u001b[39m, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 11\u001b[0m )\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kind\u001b[38;5;241m.\u001b[39mkeys():\n",
"Cell \u001b[0;32mIn[40], line 83\u001b[0m, in \u001b[0;36mupload_file_list\u001b[0;34m(deposit, flist, root, overwrite)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 82\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 83\u001b[0m existing_files \u001b[38;5;241m=\u001b[39m \u001b[43m{\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\n\u001b[1;32m 85\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdeposit\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlinks\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfiles\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mPARAMS\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 86\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
"Cell \u001b[0;32mIn[40], line 84\u001b[0m, in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 82\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 83\u001b[0m existing_files \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m---> 84\u001b[0m \u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m: f\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mget(deposit[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfiles\u001b[39m\u001b[38;5;124m\"\u001b[39m], params\u001b[38;5;241m=\u001b[39mPARAMS)\u001b[38;5;241m.\u001b[39mjson()\n\u001b[1;32m 86\u001b[0m }\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
"\u001b[0;31mTypeError\u001b[0m: string indices must be integers, not 'str'"
]
}
],
Expand All @@ -440,7 +472,7 @@
" uploads[name] += upload_file_list(\n",
" this_dep, kind[\"products\"], \"products/\", overwrite=False\n",
" )\n",
" if \"source\" in kind.keys(): \n",
" if \"source\" in kind.keys():\n",
" uploads[name] += upload_file_list(\n",
" this_dep, kind[\"source\"], \"source/\", overwrite=False\n",
" )"
Expand All @@ -449,9 +481,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:sliiders]",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "conda-env-sliiders-py"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -463,7 +495,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
"version": "3.11.8"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
Expand Down
8 changes: 7 additions & 1 deletion notebooks/shared.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

DIR_SCRATCH = Path("/tmp/ciam-scratch")

SLIIDERS_VERS = "v1.1"
SLIIDERS_VERS = "v1.2"
RES_VERS = "v1.1"

# Cloud Storage tools (will work with local storage as well but may need to be specifiec
Expand Down Expand Up @@ -223,6 +223,12 @@ def open_dataset(path, **kwargs):
return xr.open_dataset(_path, **kwargs)


def save_dataset(ds, path, **kwargs):
_path = str(_to_fuse(path))
_generate_parent_fuse_dirs(_path)
return ds.to_netcdf(_path, **kwargs)


def open_dataarray(path, **kwargs):
_path = str(_to_fuse(path))
_generate_parent_fuse_dirs(_path)
Expand Down
Loading