From bfd0aed998ac930a87cef38844b755975011b8d2 Mon Sep 17 00:00:00 2001
From: Michael Delgado <delgado.michaelt@gmail.com>
Date: Sun, 5 Jan 2020 12:26:18 -0800
Subject: [PATCH 001/224] start fresh from master, but unpin spatial packages
 from 3cef1a0

---
 base_environment.yml | 50 ++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index ea54de9..3cebbaa 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -23,7 +23,7 @@ dependencies:
 # won't work
   - esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101 # pinkeep: esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101
   - fastparquet=0.3.2=py37hc1659b7_0
-  - fiona=1.8.13=py37h900e953_0
+  - fiona=1.8.13
   - fusepy=3.0.1=py_0
 # this gcc pin is necessary b/c of a weird feature in the h553295d_15 build
 # which makes it hard to build numpy-based cython extensions (like pyclaw).
@@ -31,49 +31,49 @@ dependencies:
 # can still be built
   - gcc_linux-64=7.3.0=h553295d_14 # pinkeep: gcc_linux-64=7.3.0=h553295d_14
   - gcsfs=0.5.3=py_0
-  - gdal=3.0.2=py37hbb6b9fb_5
+  - gdal
   - geoalchemy2=0.6.3=py_0
-  - geopandas=0.6.2=py_0
-  - geopy=1.20.0=py_0
-  - geotiff=1.5.1=hbd99317_7
-  - geoviews=1.6.6=py_0
+  - geopandas
+  - geopy
+  - geotiff
+  - geoviews
   - git=2.24.0=pl526hce37bd2_1
   - gitpython=3.0.5=py_0
   - google-cloud-container=0.3.0=py37_0
   - google-cloud-storage=1.23.0=py37_0
-  - holoviews=1.12.7=py_0
-  - h5netcdf=0.7.4=py_0
-  - icu=64.2=he1b5a44_1
-  - iris=2.2.0=py37_1003
-  - jedi=0.15.1=py37_0
+  - holoviews
+  - h5netcdf
+  - icu
+  - iris
+  - jedi
 # need server proxy on workers if using remote scheduler
   - jupyter-server-proxy=1.3.2=py_0
   - kubernetes
-  - lapack=3.6.1=ha44fe06_2
-  - lz4=2.2.1=py37hd79334b_0
+  - lapack
+  - lz4
   - make=4.2.1=h14c3975_2004
   - matplotlib=3.1.2=py37_1
   - nc-time-axis=1.2.0=py_0
   - ncurses=6.1=hf484d3e_1002
-  - netcdf-fortran=4.5.2=mpi_mpich_ha8580a0_2
-  - netcdf4=1.5.3=mpi_mpich_py37h01ee55b_1
-  - numba=0.46.0=py37hb3f55d8_1
-  - numcodecs=0.6.4=py37he1b5a44_0
+  - netcdf-fortran
+  - netcdf4
+  - numba
+  - numcodecs
   - pandas=0.25.3=py37hb3f55d8_0
   # for geoviews
   - phantomjs=2.1.1=1
   - pip=19.3.1=py37_0
   - plotly=4.3.0=py_0
   - polyline=1.4.0=py_0
-  - pygeos=0.5=py37h5d51c17_1
-  - pyinterp=0.0.7=py37h97f2665_0
-  - pyshp=2.1.0=py_0
+  - pygeos
+  - pyinterp
+  - pyshp
   - python=3.7.3=h357f687_2 # pinkeep: python=3.7
-  - python-blosc=1.8.1=py37hf484d3e_0
+  - python-blosc
   - python-snappy=0.5.4=py37hee44bf9_1
   - pyviz_comms=0.7.2=py_0
   - pyyaml=5.2=py37h516909a_0
-  - rasterio=1.1.1=py37h900e953_0
+  - rasterio
   - regionmask=0.4.0=py_0
   - rtree=0.8.3=py37h7b0cdae_1003
   - scikit-image=0.16.2=py37hb3f55d8_0
@@ -82,15 +82,15 @@ dependencies:
   - seaborn=0.9.0=py_2
   # for geoviews
   - selenium=3.141.0=py37h516909a_1000
-  - shapely=1.6.4=py37h5d51c17_1007
+  - shapely
   - sparse=0.8.0=py_0
   - statsmodels=0.10.2=py37hc1659b7_0
   - tini=0.18.0=h14c3975_1001
   - unzip=6.0=h516909a_0
   - uritemplate=3.0.0=py_1
   - xarray=0.14.1=py_0
-  - xesmf=0.2.1=py_0
-  - xgcm=0.2.0=py_0
+  - xesmf
+  - xgcm
   - xhistogram=0.1.1=py_0
   - xlrd=1.2.0=py_0
   - xrft=0.2.0=py_0

From 594e67effa05dbfdd871042ead1a81778147786b Mon Sep 17 00:00:00 2001
From: Michael Delgado <delgado.michaelt@gmail.com>
Date: Sun, 5 Jan 2020 22:38:32 -0800
Subject: [PATCH 002/224] pin spatial libraries except gdal

---
 base_environment.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 3cebbaa..e82c483 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -23,7 +23,7 @@ dependencies:
 # won't work
   - esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101 # pinkeep: esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101
   - fastparquet=0.3.2=py37hc1659b7_0
-  - fiona=1.8.13
+  - fiona=1.8.13=py37h900e953_0
   - fusepy=3.0.1=py_0
 # this gcc pin is necessary b/c of a weird feature in the h553295d_15 build
 # which makes it hard to build numpy-based cython extensions (like pyclaw).
@@ -33,10 +33,10 @@ dependencies:
   - gcsfs=0.5.3=py_0
   - gdal
   - geoalchemy2=0.6.3=py_0
-  - geopandas
-  - geopy
-  - geotiff
-  - geoviews
+  - geopandas=0.6.2=py_0
+  - geopy=1.20.0=py_0
+  - geotiff=1.5.1=hbd99317_7
+  - geoviews=1.6.6=py_0
   - git=2.24.0=pl526hce37bd2_1
   - gitpython=3.0.5=py_0
   - google-cloud-container=0.3.0=py37_0
@@ -65,9 +65,9 @@ dependencies:
   - pip=19.3.1=py37_0
   - plotly=4.3.0=py_0
   - polyline=1.4.0=py_0
-  - pygeos
-  - pyinterp
-  - pyshp
+  - pygeos=0.5=py37h5d51c17_1
+  - pyinterp=0.0.7=py37h97f2665_0
+  - pyshp=2.1.0=py_0
   - python=3.7.3=h357f687_2 # pinkeep: python=3.7
   - python-blosc
   - python-snappy=0.5.4=py37hee44bf9_1
@@ -82,7 +82,7 @@ dependencies:
   - seaborn=0.9.0=py_2
   # for geoviews
   - selenium=3.141.0=py37h516909a_1000
-  - shapely
+  - shapely=1.6.4=py37h5d51c17_1007
   - sparse=0.8.0=py_0
   - statsmodels=0.10.2=py37hc1659b7_0
   - tini=0.18.0=h14c3975_1001

From bd2c37ec3aab1c4a92550062f97bcab91a9203ac Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Thu, 4 Jun 2020 12:48:22 -0700
Subject: [PATCH 003/224] test vdatum download

---
 common.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/common.sh b/common.sh
index 3078d6e..66f07ac 100644
--- a/common.sh
+++ b/common.sh
@@ -34,6 +34,11 @@ apt-get clean
 wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy
 chmod +x /usr/bin/cloud_sql_proxy
 
+# test vdatum download
+wget https://vdatum.noaa.gov/download/data/vdatum_v4.0.1.zip -O /usr/bin/vdatum.zip
+unzip /usr/bin/vdatum.zip
+rm /usr/bin/vdatum.zip
+
 # filepath curating
 chmod +x /usr/bin/prepare.sh
 mkdir /gcs

From 81cd68c0584cc026280c74a090e93cef784e6d67 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Thu, 4 Jun 2020 14:42:40 -0700
Subject: [PATCH 004/224] ians changes

---
 base_environment.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index e82c483..7843af6 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -14,6 +14,7 @@ dependencies:
   - compilers=1.0.4=0
   - dask=2.8.1=py_0
   - dask-glm=0.2.0=py_1
+  - dask_kubernetes==0.10.1
   - dask-ml=1.1.1=py_0
   - datashader=0.8.0=py_0
   - descartes=1.1.0=py_4
@@ -44,6 +45,7 @@ dependencies:
   - holoviews
   - h5netcdf
   - icu
+  - intake-esm
   - iris
   - jedi
 # need server proxy on workers if using remote scheduler
@@ -105,8 +107,3 @@ dependencies:
     - impactlab-tools==0.4.0
     - parameterize-jobs==0.1.1
     - rhg_compute_tools==0.2.2
-    - git+https://github.com/NCAR/intake-esm.git@v2020.3.16.2#egg=intake_esm
-# need to install from master until 0.10.1
-# due to handling of remote scheduler
-# (we also should at some point switch to dask-gateway instead of dask-kubernetes)
-    - dask_kubernetes==0.10.1

From 651239067b0b674414c8d9b16c5a349b29bf98ce Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Thu, 4 Jun 2020 14:43:04 -0700
Subject: [PATCH 005/224] add egm96 to vdatum

---
 common.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/common.sh b/common.sh
index 66f07ac..8678628 100644
--- a/common.sh
+++ b/common.sh
@@ -34,10 +34,13 @@ apt-get clean
 wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy
 chmod +x /usr/bin/cloud_sql_proxy
 
-# test vdatum download
+# download vdatum
 wget https://vdatum.noaa.gov/download/data/vdatum_v4.0.1.zip -O /usr/bin/vdatum.zip
+wget https://vdatum.noaa.gov/download/data/vdatum_EGM1996.zip -O /usr/bin/vdatum_EGM1996.zip
 unzip /usr/bin/vdatum.zip
+unzip -o /usr/bin/vdatum_EGM1996.zip
 rm /usr/bin/vdatum.zip
+rm /usr/bin/vdatum_EGM1996.zip
 
 # filepath curating
 chmod +x /usr/bin/prepare.sh

From 02c5d16a6e9f3cb47962d09106aa761a2407ae14 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Thu, 4 Jun 2020 14:58:54 -0700
Subject: [PATCH 006/224] use conda dask-kub name

---
 base_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base_environment.yml b/base_environment.yml
index 7843af6..21249f2 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -14,7 +14,7 @@ dependencies:
   - compilers=1.0.4=0
   - dask=2.8.1=py_0
   - dask-glm=0.2.0=py_1
-  - dask_kubernetes==0.10.1
+  - dask-kubernetes==0.10.1
   - dask-ml=1.1.1=py_0
   - datashader=0.8.0=py_0
   - descartes=1.1.0=py_4

From a7d53a00135beb8a77db39f9952f91c2ed0bb48f Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Thu, 4 Jun 2020 15:54:12 -0700
Subject: [PATCH 007/224] unpin all version numbers

---
 base_environment.yml | 156 +++++++++++++++++++++----------------------
 1 file changed, 78 insertions(+), 78 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 21249f2..93b6800 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -2,46 +2,46 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - beautifulsoup4=4.8.1=py37_0
-  - bottleneck=1.3.1=py37hc1659b7_0
-  - boto3=1.10.34=py_0
-  - bqplot=0.12.1=py_0
-  - bumpversion=0.5.3=py_1001
-  - cachecontrol=0.12.5=py_0
-  - cartopy=0.17.0=py37h423102d_1009
-  - cftime=1.0.4.2=py37hc1659b7_0
-  - click=7.0=py_0
-  - compilers=1.0.4=0
-  - dask=2.8.1=py_0
-  - dask-glm=0.2.0=py_1
-  - dask-kubernetes==0.10.1
-  - dask-ml=1.1.1=py_0
-  - datashader=0.8.0=py_0
-  - descartes=1.1.0=py_4
-  - distributed=2.8.1=py_0
-  - dropbox=9.4.0=py_0
+  - beautifulsoup4
+  - bottleneck
+  - boto3
+  - bqplot
+  - bumpversion
+  - cachecontrol
+  - cartopy
+  - cftime
+  - click
+  - compilers
+  - dask
+  - dask-glm
+  - dask-kubernetes
+  - dask-ml
+  - datashader
+  - descartes
+  - distributed
+  - dropbox
 # need to make sure we get esmpy compiled with mpi otherwise xesmf regridding
 # won't work
-  - esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101 # pinkeep: esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101
-  - fastparquet=0.3.2=py37hc1659b7_0
-  - fiona=1.8.13=py37h900e953_0
-  - fusepy=3.0.1=py_0
+  - esmpy
+  - fastparquet
+  - fiona
+  - fusepy
 # this gcc pin is necessary b/c of a weird feature in the h553295d_15 build
 # which makes it hard to build numpy-based cython extensions (like pyclaw).
 # we should try removing it whenever we next do an update and see if Clawpack
 # can still be built
-  - gcc_linux-64=7.3.0=h553295d_14 # pinkeep: gcc_linux-64=7.3.0=h553295d_14
-  - gcsfs=0.5.3=py_0
-  - gdal
-  - geoalchemy2=0.6.3=py_0
-  - geopandas=0.6.2=py_0
-  - geopy=1.20.0=py_0
-  - geotiff=1.5.1=hbd99317_7
-  - geoviews=1.6.6=py_0
-  - git=2.24.0=pl526hce37bd2_1
-  - gitpython=3.0.5=py_0
-  - google-cloud-container=0.3.0=py37_0
-  - google-cloud-storage=1.23.0=py37_0
+  - gcc_linux-64
+  - gcsfs
+  - gda
+  - geoalchemy2
+  - geopandas
+  - geopy
+  - geotiff
+  - geoviews
+  - git
+  - gitpython
+  - google-cloud-container
+  - google-cloud-storage
   - holoviews
   - h5netcdf
   - icu
@@ -49,61 +49,61 @@ dependencies:
   - iris
   - jedi
 # need server proxy on workers if using remote scheduler
-  - jupyter-server-proxy=1.3.2=py_0
+  - jupyter-server-proxy
   - kubernetes
   - lapack
   - lz4
-  - make=4.2.1=h14c3975_2004
-  - matplotlib=3.1.2=py37_1
-  - nc-time-axis=1.2.0=py_0
-  - ncurses=6.1=hf484d3e_1002
+  - make
+  - matplotlib
+  - nc-time-axis
+  - ncurses
   - netcdf-fortran
   - netcdf4
   - numba
   - numcodecs
-  - pandas=0.25.3=py37hb3f55d8_0
+  - pandas
   # for geoviews
-  - phantomjs=2.1.1=1
-  - pip=19.3.1=py37_0
-  - plotly=4.3.0=py_0
-  - polyline=1.4.0=py_0
-  - pygeos=0.5=py37h5d51c17_1
-  - pyinterp=0.0.7=py37h97f2665_0
-  - pyshp=2.1.0=py_0
-  - python=3.7.3=h357f687_2 # pinkeep: python=3.7
+  - phantomjs
+  - pip
+  - plotly
+  - polyline
+  - pygeos
+  - pyinterp
+  - pyshp
+  - python
   - python-blosc
-  - python-snappy=0.5.4=py37hee44bf9_1
-  - pyviz_comms=0.7.2=py_0
-  - pyyaml=5.2=py37h516909a_0
+  - python-snappy
+  - pyviz_comms
+  - pyyaml
   - rasterio
-  - regionmask=0.4.0=py_0
-  - rtree=0.8.3=py37h7b0cdae_1003
-  - scikit-image=0.16.2=py37hb3f55d8_0
-  - scikit-learn=0.22=py37hcdab131_0
-  - scipy=1.3.2=py37h921218d_0
-  - seaborn=0.9.0=py_2
+  - regionmask
+  - rtree
+  - scikit-image
+  - scikit-learn
+  - scipy
+  - seaborn
   # for geoviews
-  - selenium=3.141.0=py37h516909a_1000
-  - shapely=1.6.4=py37h5d51c17_1007
-  - sparse=0.8.0=py_0
-  - statsmodels=0.10.2=py37hc1659b7_0
-  - tini=0.18.0=h14c3975_1001
-  - unzip=6.0=h516909a_0
-  - uritemplate=3.0.0=py_1
-  - xarray=0.14.1=py_0
+  - selenium
+  - shapely
+  - sparse
+  - statsmodels
+  - tini
+  - unzip
+  - uritemplate
+  - xarray
   - xesmf
   - xgcm
-  - xhistogram=0.1.1=py_0
-  - xlrd=1.2.0=py_0
-  - xrft=0.2.0=py_0
-  - zarr=2.3.2=py37_0
-  - zeromq=4.3.2=he1b5a44_2
-  - zict=1.0.0=py_0
+  - xhistogram
+  - xlrd
+  - xrft
+  - zarr
+  - zeromq
+  - zict
   - pip:
-    - mapbox==0.18.0
-    - py-noaa==1.0
-    - sidecar==0.3.0
-    - climate-toolbox==0.1.5
-    - impactlab-tools==0.4.0
-    - parameterize-jobs==0.1.1
-    - rhg_compute_tools==0.2.2
+    - mapbox
+    - py-noaa
+    - sidecar
+    - climate-toolbox
+    - impactlab-tools
+    - parameterize-jobs
+    - rhg_compute_tools

From 2317167b3e030635057dfe321ad31d941755f52b Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Thu, 4 Jun 2020 16:00:47 -0700
Subject: [PATCH 008/224] fix typo

---
 base_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base_environment.yml b/base_environment.yml
index 93b6800..fd4d1b5 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -32,7 +32,7 @@ dependencies:
 # can still be built
   - gcc_linux-64
   - gcsfs
-  - gda
+  - gdal
   - geoalchemy2
   - geopandas
   - geopy

From 6893ff092a23a17643591b95be5e60ed077126d9 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Thu, 4 Jun 2020 16:11:49 -0700
Subject: [PATCH 009/224] install unzip before unzipping

---
 common.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/common.sh b/common.sh
index 8678628..f54e9a6 100644
--- a/common.sh
+++ b/common.sh
@@ -5,6 +5,7 @@ apt-get install -yq --no-install-recommends \
   apt-utils \
   zip \
   bzip2 \
+  unzip \
   ca-certificates \
   curl \
   lsb-release \

From 210d2767dae385b63cf06e190fcafaffdabec0a7 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Thu, 4 Jun 2020 16:18:21 -0700
Subject: [PATCH 010/224] drop unnecessary packages

---
 base_environment.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index fd4d1b5..b3a05cd 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -100,10 +100,8 @@ dependencies:
   - zeromq
   - zict
   - pip:
-    - mapbox
     - py-noaa
     - sidecar
     - climate-toolbox
-    - impactlab-tools
     - parameterize-jobs
     - rhg_compute_tools

From 26ed42fbed0c6f42c96cd3f6e7352a36909cbe74 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Thu, 4 Jun 2020 17:38:22 -0700
Subject: [PATCH 011/224] use minor versions from resolved conda travis output

---
 base_environment.yml | 201 +++++++++++++++++++++----------------------
 1 file changed, 96 insertions(+), 105 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index b3a05cd..36b5a06 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,107 +1,98 @@
-name: base
 channels:
-  - conda-forge
+- conda-forge
 dependencies:
-  - beautifulsoup4
-  - bottleneck
-  - boto3
-  - bqplot
-  - bumpversion
-  - cachecontrol
-  - cartopy
-  - cftime
-  - click
-  - compilers
-  - dask
-  - dask-glm
-  - dask-kubernetes
-  - dask-ml
-  - datashader
-  - descartes
-  - distributed
-  - dropbox
-# need to make sure we get esmpy compiled with mpi otherwise xesmf regridding
-# won't work
-  - esmpy
-  - fastparquet
-  - fiona
-  - fusepy
-# this gcc pin is necessary b/c of a weird feature in the h553295d_15 build
-# which makes it hard to build numpy-based cython extensions (like pyclaw).
-# we should try removing it whenever we next do an update and see if Clawpack
-# can still be built
-  - gcc_linux-64
-  - gcsfs
-  - gdal
-  - geoalchemy2
-  - geopandas
-  - geopy
-  - geotiff
-  - geoviews
-  - git
-  - gitpython
-  - google-cloud-container
-  - google-cloud-storage
-  - holoviews
-  - h5netcdf
-  - icu
-  - intake-esm
-  - iris
-  - jedi
-# need server proxy on workers if using remote scheduler
-  - jupyter-server-proxy
-  - kubernetes
-  - lapack
-  - lz4
-  - make
-  - matplotlib
-  - nc-time-axis
-  - ncurses
-  - netcdf-fortran
-  - netcdf4
-  - numba
-  - numcodecs
-  - pandas
-  # for geoviews
-  - phantomjs
-  - pip
-  - plotly
-  - polyline
-  - pygeos
-  - pyinterp
-  - pyshp
-  - python
-  - python-blosc
-  - python-snappy
-  - pyviz_comms
-  - pyyaml
-  - rasterio
-  - regionmask
-  - rtree
-  - scikit-image
-  - scikit-learn
-  - scipy
-  - seaborn
-  # for geoviews
-  - selenium
-  - shapely
-  - sparse
-  - statsmodels
-  - tini
-  - unzip
-  - uritemplate
-  - xarray
-  - xesmf
-  - xgcm
-  - xhistogram
-  - xlrd
-  - xrft
-  - zarr
-  - zeromq
-  - zict
-  - pip:
-    - py-noaa
-    - sidecar
-    - climate-toolbox
-    - parameterize-jobs
-    - rhg_compute_tools
+- beautifulsoup4=4.9
+- bottleneck=1.3
+- boto3=1.13
+- bqplot=0.12
+- bumpversion=0.5
+- cachecontrol=0.12
+- cartopy=0.17
+- cftime=1.1
+- click=7.1
+- compilers=1.0
+- dask=2.17
+- dask-glm=0.2
+- dask-kubernetes=0.10
+- dask-ml=1.5
+- datashader=0.11
+- descartes=1.1
+- distributed=2.17
+- dropbox=9.4
+- esmpy=7.1
+- fastparquet=0.4
+- fiona=1.8
+- fusepy=3.0
+- gcc_linux-64=7.5
+- gcsfs=0.6
+- gdal=3.0
+- geoalchemy2=0.6
+- geopandas=0.7
+- geopy=1.22
+- geotiff=1.5
+- geoviews=1.8
+- git=2.26
+- gitpython=3.1
+- google-cloud-container=0.3
+- google-cloud-storage=1.28
+- holoviews=1.13
+- h5netcdf=0.8
+- icu=58.2
+- intake-esm=2020.5
+- iris=2.4
+- jedi=0.17
+- jupyter-server-proxy=1.5
+- kubernetes=1.18
+- lapack=3.6
+- lz4=3.0
+- make=4.3
+- matplotlib=3.1
+- nc-time-axis=1.2
+- ncurses=6.1
+- netcdf-fortran=4.4
+- netcdf4=1.5
+- numba=0.48
+- numcodecs=0.6
+- pandas=1.0
+- phantomjs=2.1
+- pip=19.3
+- plotly=4.8
+- polyline=1.4
+- pygeos=0.6
+- pyinterp=0.3
+- pyshp=2.1
+- python=3.7
+- python-blosc=1.9
+- python-snappy=0.5
+- pyviz_comms=0.7
+- pyyaml=5.3
+- rasterio=1.1
+- regionmask=0.5
+- rtree=0.9
+- scikit-image=0.17
+- scikit-learn=0.23
+- scipy=1.4
+- seaborn=0.10
+- selenium=3.141
+- shapely=1.7
+- sparse=0.9
+- statsmodels=0.11
+- tini=0.18
+- unzip=6.0
+- uritemplate=3.0
+- xarray=0.15
+- xesmf=0.3
+- xgcm=0.3
+- xhistogram=0.1
+- xlrd=1.2
+- xrft=0.2
+- zarr=2.4
+- zeromq=4.3
+- zict=2.0
+- pip:
+  - py-noaa
+  - sidecar
+  - climate-toolbox
+  - parameterize-jobs
+  - rhg_compute_tools
+name: base

From 01916c1e46f8a5ee64dc85db00ffc967b38ab2dc Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 5 Jun 2020 09:48:36 -0700
Subject: [PATCH 012/224] drop some packages

---
 base_environment.yml | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 36b5a06..a5f6a97 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -5,7 +5,7 @@ dependencies:
 - bottleneck=1.3
 - boto3=1.13
 - bqplot=0.12
-- bumpversion=0.5
+- bump2version
 - cachecontrol=0.12
 - cartopy=0.17
 - cftime=1.1
@@ -16,17 +16,14 @@ dependencies:
 - dask-kubernetes=0.10
 - dask-ml=1.5
 - datashader=0.11
-- descartes=1.1
 - distributed=2.17
 - dropbox=9.4
 - esmpy=7.1
 - fastparquet=0.4
 - fiona=1.8
 - fusepy=3.0
-- gcc_linux-64=7.5
 - gcsfs=0.6
 - gdal=3.0
-- geoalchemy2=0.6
 - geopandas=0.7
 - geopy=1.22
 - geotiff=1.5
@@ -78,17 +75,12 @@ dependencies:
 - sparse=0.9
 - statsmodels=0.11
 - tini=0.18
-- unzip=6.0
-- uritemplate=3.0
 - xarray=0.15
 - xesmf=0.3
-- xgcm=0.3
 - xhistogram=0.1
 - xlrd=1.2
-- xrft=0.2
 - zarr=2.4
 - zeromq=4.3
-- zict=2.0
 - pip:
   - py-noaa
   - sidecar

From 86584df60b963c4a02d78d2f2671308cde520f46 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 5 Jun 2020 09:51:10 -0700
Subject: [PATCH 013/224] drop fusepy

---
 base_environment.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/base_environment.yml b/base_environment.yml
index a5f6a97..e2d740c 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -21,7 +21,6 @@ dependencies:
 - esmpy=7.1
 - fastparquet=0.4
 - fiona=1.8
-- fusepy=3.0
 - gcsfs=0.6
 - gdal=3.0
 - geopandas=0.7

From 5f673d9127d258400b8a2ae79d4926c7d26b0203 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 5 Jun 2020 09:59:23 -0700
Subject: [PATCH 014/224] more drops

---
 base_environment.yml | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index e2d740c..948f648 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -2,6 +2,7 @@ channels:
 - conda-forge
 dependencies:
 - beautifulsoup4=4.9
+- blas
 - bottleneck=1.3
 - boto3=1.13
 - bqplot=0.12
@@ -24,7 +25,6 @@ dependencies:
 - gcsfs=0.6
 - gdal=3.0
 - geopandas=0.7
-- geopy=1.22
 - geotiff=1.5
 - geoviews=1.8
 - git=2.26
@@ -33,34 +33,25 @@ dependencies:
 - google-cloud-storage=1.28
 - holoviews=1.13
 - h5netcdf=0.8
-- icu=58.2
 - intake-esm=2020.5
-- iris=2.4
 - jedi=0.17
 - jupyter-server-proxy=1.5
 - kubernetes=1.18
-- lapack=3.6
 - lz4=3.0
 - make=4.3
 - matplotlib=3.1
 - nc-time-axis=1.2
-- ncurses=6.1
 - netcdf-fortran=4.4
 - netcdf4=1.5
 - numba=0.48
-- numcodecs=0.6
 - pandas=1.0
-- phantomjs=2.1
+- phantomjs=2.1 # optional dependency of geoviews
+- pillow  # optional dependency of geoviews
 - pip=19.3
 - plotly=4.8
-- polyline=1.4
-- pygeos=0.6
 - pyinterp=0.3
 - pyshp=2.1
 - python=3.7
-- python-blosc=1.9
-- python-snappy=0.5
-- pyviz_comms=0.7
 - pyyaml=5.3
 - rasterio=1.1
 - regionmask=0.5
@@ -69,7 +60,7 @@ dependencies:
 - scikit-learn=0.23
 - scipy=1.4
 - seaborn=0.10
-- selenium=3.141
+- selenium=3.141 # optional dependency of geoviews
 - shapely=1.7
 - sparse=0.9
 - statsmodels=0.11

From 8f46c895f0f964f9d88d14e7d5acb2b644593c67 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Tue, 9 Jun 2020 14:15:46 -0700
Subject: [PATCH 015/224] add climada

---
 base_environment.yml | 136 +++++++++++++++++++++----------------------
 common.sh            |   7 +++
 2 files changed, 75 insertions(+), 68 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 948f648..beef353 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,76 +1,76 @@
 channels:
 - conda-forge
 dependencies:
-- beautifulsoup4=4.9
+- beautifulsoup4
 - blas
-- bottleneck=1.3
-- boto3=1.13
-- bqplot=0.12
+- bottleneck
+- boto3
+- bqplot
 - bump2version
-- cachecontrol=0.12
-- cartopy=0.17
-- cftime=1.1
-- click=7.1
-- compilers=1.0
-- dask=2.17
-- dask-glm=0.2
-- dask-kubernetes=0.10
-- dask-ml=1.5
-- datashader=0.11
-- distributed=2.17
-- dropbox=9.4
-- esmpy=7.1
-- fastparquet=0.4
-- fiona=1.8
-- gcsfs=0.6
-- gdal=3.0
-- geopandas=0.7
-- geotiff=1.5
-- geoviews=1.8
-- git=2.26
-- gitpython=3.1
-- google-cloud-container=0.3
-- google-cloud-storage=1.28
-- holoviews=1.13
-- h5netcdf=0.8
-- intake-esm=2020.5
-- jedi=0.17
-- jupyter-server-proxy=1.5
-- kubernetes=1.18
-- lz4=3.0
-- make=4.3
-- matplotlib=3.1
-- nc-time-axis=1.2
-- netcdf-fortran=4.4
-- netcdf4=1.5
-- numba=0.48
-- pandas=1.0
-- phantomjs=2.1 # optional dependency of geoviews
-- pillow  # optional dependency of geoviews
-- pip=19.3
-- plotly=4.8
-- pyinterp=0.3
-- pyshp=2.1
-- python=3.7
-- pyyaml=5.3
-- rasterio=1.1
-- regionmask=0.5
-- rtree=0.9
-- scikit-image=0.17
-- scikit-learn=0.23
-- scipy=1.4
-- seaborn=0.10
-- selenium=3.141 # optional dependency of geoviews
-- shapely=1.7
-- sparse=0.9
-- statsmodels=0.11
-- tini=0.18
-- xarray=0.15
-- xesmf=0.3
-- xhistogram=0.1
-- xlrd=1.2
-- zarr=2.4
-- zeromq=4.3
+- cachecontrol
+- cartopy
+- cftime
+- click
+- compilers
+- dask
+- dask-glm
+- dask-kubernetes
+- dask-ml
+- datashader
+- distributed
+- dropbox
+- esmpy
+- fastparquet
+- fiona
+- gcsfs
+- gdal
+- geopandas
+- geotiff
+- geoviews
+- git
+- gitpython
+- google-cloud-container
+- google-cloud-storage
+- holoviews
+- h5netcdf
+- intake-esm
+- jedi
+- jupyter-server-proxy
+- kubernetes
+- lz4
+- make
+- matplotlib
+- nc-time-axis
+- netcdf-fortran
+- netcdf4
+- numba
+- pandas
+- phantomjs # optional dependency of geoviews
+- pillow # optional dependency of geoviews
+- pip
+- plotly
+- pyinterp
+- pyshp
+- python
+- pyyaml
+- rasterio
+- regionmask
+- rtree
+- scikit-image
+- scikit-learn
+- scipy
+- seaborn
+- selenium # optional dependency of geoviews
+- shapely
+- sparse
+- statsmodels
+- tini
+- xarray
+- xesmf
+- xhistogram
+- xlrd
+- zarr
+- zeromq
 - pip:
   - py-noaa
   - sidecar
diff --git a/common.sh b/common.sh
index f54e9a6..e47273c 100644
--- a/common.sh
+++ b/common.sh
@@ -43,6 +43,13 @@ unzip -o /usr/bin/vdatum_EGM1996.zip
 rm /usr/bin/vdatum.zip
 rm /usr/bin/vdatum_EGM1996.zip
 
+# download climada
+wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
+unzip /usr/bin/climada_python-1.4.2.zip
+rm /usr/bin/climada_python-1.4.2.zip
+sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
+sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
+
 # filepath curating
 chmod +x /usr/bin/prepare.sh
 mkdir /gcs

From e59162b6f33b1f11125770c4fd4ed9a1a9415fae Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Tue, 9 Jun 2020 14:23:50 -0700
Subject: [PATCH 016/224] fix paths

---
 common.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common.sh b/common.sh
index e47273c..10ea39f 100644
--- a/common.sh
+++ b/common.sh
@@ -47,8 +47,8 @@ rm /usr/bin/vdatum_EGM1996.zip
 wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
 unzip /usr/bin/climada_python-1.4.2.zip
 rm /usr/bin/climada_python-1.4.2.zip
-sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
-sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
+sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
+sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
 
 # filepath curating
 chmod +x /usr/bin/prepare.sh

From 31128a56849bb2fcdd7258be21ef7af4e0282a28 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Tue, 9 Jun 2020 15:00:34 -0700
Subject: [PATCH 017/224] try separate env instead

---
 common.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/common.sh b/common.sh
index 10ea39f..418be39 100644
--- a/common.sh
+++ b/common.sh
@@ -47,8 +47,7 @@ rm /usr/bin/vdatum_EGM1996.zip
 wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
 unzip /usr/bin/climada_python-1.4.2.zip
 rm /usr/bin/climada_python-1.4.2.zip
-sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
-sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
+conda env create -f /usr/bin/climada_python-1.4.2/requirements/env_climada.yml --name climada_env
 
 # filepath curating
 chmod +x /usr/bin/prepare.sh

From 07a36243e6f03eb3d28d33324516f811624982aa Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 12:01:47 -0700
Subject: [PATCH 018/224] Revert "try separate env instead"

This reverts commit cf43aa6dd3cd00daf7618ba4f1fc3e1bfc55ef1a.
---
 common.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/common.sh b/common.sh
index 418be39..10ea39f 100644
--- a/common.sh
+++ b/common.sh
@@ -47,7 +47,8 @@ rm /usr/bin/vdatum_EGM1996.zip
 wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
 unzip /usr/bin/climada_python-1.4.2.zip
 rm /usr/bin/climada_python-1.4.2.zip
-conda env create -f /usr/bin/climada_python-1.4.2/requirements/env_climada.yml --name climada_env
+sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
+sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
 
 # filepath curating
 chmod +x /usr/bin/prepare.sh

From a13aae3cb951073fa2dee8a1b182dd3534f1e17c Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 12:01:50 -0700
Subject: [PATCH 019/224] Revert "fix paths"

This reverts commit 2dea811a821b2a8650d1c0979fffba181adaa6af.
---
 common.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common.sh b/common.sh
index 10ea39f..e47273c 100644
--- a/common.sh
+++ b/common.sh
@@ -47,8 +47,8 @@ rm /usr/bin/vdatum_EGM1996.zip
 wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
 unzip /usr/bin/climada_python-1.4.2.zip
 rm /usr/bin/climada_python-1.4.2.zip
-sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
-sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
+sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
+sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
 
 # filepath curating
 chmod +x /usr/bin/prepare.sh

From f1a2f4b76b2ae9bd6cf017e8d81ea553c6be1be6 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 12:01:51 -0700
Subject: [PATCH 020/224] Revert "add climada"

This reverts commit 1d581250be59f248786394c15723e474c765c21f.
---
 base_environment.yml | 136 +++++++++++++++++++++----------------------
 common.sh            |   7 ---
 2 files changed, 68 insertions(+), 75 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index beef353..948f648 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,76 +1,76 @@
 channels:
 - conda-forge
 dependencies:
-- beautifulsoup4
+- beautifulsoup4=4.9
 - blas
-- bottleneck
-- boto3
-- bqplot
+- bottleneck=1.3
+- boto3=1.13
+- bqplot=0.12
 - bump2version
-- cachecontrol
-- cartopy
-- cftime
-- click
-- compilers
-- dask
-- dask-glm
-- dask-kubernetes
-- dask-ml
-- datashader
-- distributed
-- dropbox
-- esmpy
-- fastparquet
-- fiona
-- gcsfs
-- gdal
-- geopandas
-- geotiff
-- geoviews
-- git
-- gitpython
-- google-cloud-container
-- google-cloud-storage
-- holoviews
-- h5netcdf
-- intake-esm
-- jedi
-- jupyter-server-proxy
-- kubernetes
-- lz4
-- make
-- matplotlib
-- nc-time-axis
-- netcdf-fortran
-- netcdf4
-- numba
-- pandas
-- phantomjs # optional dependency of geoviews
-- pillow # optional dependency of geoviews
-- pip
-- plotly
-- pyinterp
-- pyshp
-- python
-- pyyaml
-- rasterio
-- regionmask
-- rtree
-- scikit-image
-- scikit-learn
-- scipy
-- seaborn
-- selenium # optional dependency of geoviews
-- shapely
-- sparse
-- statsmodels
-- tini
-- xarray
-- xesmf
-- xhistogram
-- xlrd
-- zarr
-- zeromq
+- cachecontrol=0.12
+- cartopy=0.17
+- cftime=1.1
+- click=7.1
+- compilers=1.0
+- dask=2.17
+- dask-glm=0.2
+- dask-kubernetes=0.10
+- dask-ml=1.5
+- datashader=0.11
+- distributed=2.17
+- dropbox=9.4
+- esmpy=7.1
+- fastparquet=0.4
+- fiona=1.8
+- gcsfs=0.6
+- gdal=3.0
+- geopandas=0.7
+- geotiff=1.5
+- geoviews=1.8
+- git=2.26
+- gitpython=3.1
+- google-cloud-container=0.3
+- google-cloud-storage=1.28
+- holoviews=1.13
+- h5netcdf=0.8
+- intake-esm=2020.5
+- jedi=0.17
+- jupyter-server-proxy=1.5
+- kubernetes=1.18
+- lz4=3.0
+- make=4.3
+- matplotlib=3.1
+- nc-time-axis=1.2
+- netcdf-fortran=4.4
+- netcdf4=1.5
+- numba=0.48
+- pandas=1.0
+- phantomjs=2.1 # optional dependency of geoviews
+- pillow  # optional dependency of geoviews
+- pip=19.3
+- plotly=4.8
+- pyinterp=0.3
+- pyshp=2.1
+- python=3.7
+- pyyaml=5.3
+- rasterio=1.1
+- regionmask=0.5
+- rtree=0.9
+- scikit-image=0.17
+- scikit-learn=0.23
+- scipy=1.4
+- seaborn=0.10
+- selenium=3.141 # optional dependency of geoviews
+- shapely=1.7
+- sparse=0.9
+- statsmodels=0.11
+- tini=0.18
+- xarray=0.15
+- xesmf=0.3
+- xhistogram=0.1
+- xlrd=1.2
+- zarr=2.4
+- zeromq=4.3
 - pip:
   - py-noaa
   - sidecar
diff --git a/common.sh b/common.sh
index e47273c..f54e9a6 100644
--- a/common.sh
+++ b/common.sh
@@ -43,13 +43,6 @@ unzip -o /usr/bin/vdatum_EGM1996.zip
 rm /usr/bin/vdatum.zip
 rm /usr/bin/vdatum_EGM1996.zip
 
-# download climada
-wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
-unzip /usr/bin/climada_python-1.4.2.zip
-rm /usr/bin/climada_python-1.4.2.zip
-sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
-sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
-
 # filepath curating
 chmod +x /usr/bin/prepare.sh
 mkdir /gcs

From c698d57c4f76d9f34fa434cd7d8ddbb43edeea75 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 12:03:31 -0700
Subject: [PATCH 021/224] unpin everything

---
 base_environment.yml | 134 +++++++++++++++++++++----------------------
 1 file changed, 67 insertions(+), 67 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 948f648..7bae900 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,76 +1,76 @@
 channels:
 - conda-forge
 dependencies:
-- beautifulsoup4=4.9
+- beautifulsoup4
 - blas
-- bottleneck=1.3
-- boto3=1.13
-- bqplot=0.12
+- bottleneck
+- boto3
+- bqplot
 - bump2version
-- cachecontrol=0.12
-- cartopy=0.17
-- cftime=1.1
-- click=7.1
-- compilers=1.0
-- dask=2.17
-- dask-glm=0.2
-- dask-kubernetes=0.10
-- dask-ml=1.5
-- datashader=0.11
-- distributed=2.17
-- dropbox=9.4
-- esmpy=7.1
-- fastparquet=0.4
-- fiona=1.8
-- gcsfs=0.6
-- gdal=3.0
-- geopandas=0.7
-- geotiff=1.5
-- geoviews=1.8
-- git=2.26
-- gitpython=3.1
-- google-cloud-container=0.3
-- google-cloud-storage=1.28
-- holoviews=1.13
-- h5netcdf=0.8
-- intake-esm=2020.5
-- jedi=0.17
-- jupyter-server-proxy=1.5
-- kubernetes=1.18
-- lz4=3.0
-- make=4.3
-- matplotlib=3.1
-- nc-time-axis=1.2
-- netcdf-fortran=4.4
-- netcdf4=1.5
-- numba=0.48
-- pandas=1.0
-- phantomjs=2.1 # optional dependency of geoviews
+- cachecontrol
+- cartopy
+- cftime
+- click
+- compilers
+- dask
+- dask-glm
+- dask-kubernetes
+- dask-ml
+- datashader
+- distributed
+- dropbox
+- esmpy
+- fastparquet
+- fiona
+- gcsfs
+- gdal
+- geopandas
+- geotiff
+- geoviews
+- git
+- gitpython
+- google-cloud-container
+- google-cloud-storage
+- holoviews
+- h5netcdf
+- intake-esm
+- jedi
+- jupyter-server-proxy
+- kubernetes
+- lz4
+- make
+- matplotlib
+- nc-time-axis
+- netcdf-fortran
+- netcdf4
+- numba
+- pandas
+- phantomjs # optional dependency of geoviews
 - pillow  # optional dependency of geoviews
-- pip=19.3
-- plotly=4.8
-- pyinterp=0.3
-- pyshp=2.1
-- python=3.7
-- pyyaml=5.3
-- rasterio=1.1
-- regionmask=0.5
-- rtree=0.9
-- scikit-image=0.17
-- scikit-learn=0.23
-- scipy=1.4
-- seaborn=0.10
-- selenium=3.141 # optional dependency of geoviews
-- shapely=1.7
-- sparse=0.9
-- statsmodels=0.11
-- tini=0.18
-- xarray=0.15
-- xesmf=0.3
-- xhistogram=0.1
-- xlrd=1.2
-- zarr=2.4
-- zeromq=4.3
+- pip
+- plotly
+- pyinterp
+- pyshp
+- python
+- pyyaml
+- rasterio
+- regionmask
+- rtree
+- scikit-image
+- scikit-learn
+- scipy
+- seaborn
+- selenium # optional dependency of geoviews
+- shapely
+- sparse
+- statsmodels
+- tini
+- xarray
+- xesmf
+- xhistogram
+- xlrd
+- zarr
+- zeromq
 - pip:
   - py-noaa
   - sidecar

From 4664ead6aea6174de7ee132750ff895f3f5ccf89 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 12:29:34 -0700
Subject: [PATCH 022/224] tabs to match earlier commit

---
 base_environment.yml | 154 +++++++++++++++++++++----------------------
 1 file changed, 77 insertions(+), 77 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 7bae900..36fd320 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,80 +1,80 @@
 channels:
-- conda-forge
+  - conda-forge
 dependencies:
-- beautifulsoup4
-- blas
-- bottleneck
-- boto3
-- bqplot
-- bump2version
-- cachecontrol
-- cartopy
-- cftime
-- click
-- compilers
-- dask
-- dask-glm
-- dask-kubernetes
-- dask-ml
-- datashader
-- distributed
-- dropbox
-- esmpy
-- fastparquet
-- fiona
-- gcsfs
-- gdal
-- geopandas
-- geotiff
-- geoviews
-- git
-- gitpython
-- google-cloud-container
-- google-cloud-storage
-- holoviews
-- h5netcdf
-- intake-esm
-- jedi
-- jupyter-server-proxy
-- kubernetes
-- lz4
-- make
-- matplotlib
-- nc-time-axis
-- netcdf-fortran
-- netcdf4
-- numba
-- pandas
-- phantomjs # optional dependency of geoviews
-- pillow  # optional dependency of geoviews
-- pip
-- plotly
-- pyinterp
-- pyshp
-- python
-- pyyaml
-- rasterio
-- regionmask
-- rtree
-- scikit-image
-- scikit-learn
-- scipy
-- seaborn
-- selenium # optional dependency of geoviews
-- shapely
-- sparse
-- statsmodels
-- tini
-- xarray
-- xesmf
-- xhistogram
-- xlrd
-- zarr
-- zeromq
-- pip:
-  - py-noaa
-  - sidecar
-  - climate-toolbox
-  - parameterize-jobs
-  - rhg_compute_tools
+  - beautifulsoup4
+  - blas
+  - bottleneck
+  - boto3
+  - bqplot
+  - bump2version
+  - cachecontrol
+  - cartopy
+  - cftime
+  - click
+  - compilers
+  - dask
+  - dask-glm
+  - dask-kubernetes
+  - dask-ml
+  - datashader
+  - distributed
+  - dropbox
+  - esmpy
+  - fastparquet
+  - fiona
+  - gcsfs
+  - gdal
+  - geopandas
+  - geotiff
+  - geoviews
+  - git
+  - gitpython
+  - google-cloud-container
+  - google-cloud-storage
+  - holoviews
+  - h5netcdf
+  - intake-esm
+  - jedi
+  - jupyter-server-proxy
+  - kubernetes
+  - lz4
+  - make
+  - matplotlib
+  - nc-time-axis
+  - netcdf-fortran
+  - netcdf4
+  - numba
+  - pandas
+  - phantomjs # optional dependency of geoviews
+  - pillow  # optional dependency of geoviews
+  - pip
+  - plotly
+  - pyinterp
+  - pyshp
+  - python
+  - pyyaml
+  - rasterio
+  - regionmask
+  - rtree
+  - scikit-image
+  - scikit-learn
+  - scipy
+  - seaborn
+  - selenium # optional dependency of geoviews
+  - shapely
+  - sparse
+  - statsmodels
+  - tini
+  - xarray
+  - xesmf
+  - xhistogram
+  - xlrd
+  - zarr
+  - zeromq
+  - pip:
+    - py-noaa
+    - sidecar
+    - climate-toolbox
+    - parameterize-jobs
+    - rhg_compute_tools
 name: base

From 46585fa9793181577126f352e2d1fea53db33c10 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 12:31:57 -0700
Subject: [PATCH 023/224] remove packages added since last commit where solver
 worked

---
 base_environment.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 36fd320..1ee3ebd 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -2,11 +2,11 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4
-  - blas
+  # - blas
   - bottleneck
   - boto3
   - bqplot
-  - bump2version
+  # - bump2version
   - cachecontrol
   - cartopy
   - cftime
@@ -46,7 +46,7 @@ dependencies:
   - numba
   - pandas
   - phantomjs # optional dependency of geoviews
-  - pillow  # optional dependency of geoviews
+  # - pillow  # optional dependency of geoviews
   - pip
   - plotly
   - pyinterp

From 29067da6dd217b7fb23b40b1f73067810db8f915 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 12:54:12 -0700
Subject: [PATCH 024/224] Revert "drop unnecessary packages"

This reverts commit 879b5bee7d8138d0ef3fa21f5b1363e98ce9e43e.
---
 base_environment.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/base_environment.yml b/base_environment.yml
index 1ee3ebd..52b3310 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -72,9 +72,11 @@ dependencies:
   - zarr
   - zeromq
   - pip:
+    - mapbox
     - py-noaa
     - sidecar
     - climate-toolbox
+    - impactlab-tools
     - parameterize-jobs
     - rhg_compute_tools
 name: base

From 95b7928a2a8a293ad2c1847bcd1cd08e36877c08 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 12:55:43 -0700
Subject: [PATCH 025/224] undo revert

---
 base_environment.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 52b3310..1ee3ebd 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -72,11 +72,9 @@ dependencies:
   - zarr
   - zeromq
   - pip:
-    - mapbox
     - py-noaa
     - sidecar
     - climate-toolbox
-    - impactlab-tools
     - parameterize-jobs
     - rhg_compute_tools
 name: base

From 44f72fed85b7770396a53fc13006feaf36b81fec Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:00:44 -0700
Subject: [PATCH 026/224] Revert "drop unnecessary packages"

This reverts commit 879b5bee7d8138d0ef3fa21f5b1363e98ce9e43e.
---
 base_environment.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/base_environment.yml b/base_environment.yml
index 1ee3ebd..52b3310 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -72,9 +72,11 @@ dependencies:
   - zarr
   - zeromq
   - pip:
+    - mapbox
     - py-noaa
     - sidecar
     - climate-toolbox
+    - impactlab-tools
     - parameterize-jobs
     - rhg_compute_tools
 name: base

From c429a4f437acf4a81d33d0f63e8e6c8a03ce486a Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:14 -0700
Subject: [PATCH 027/224] Revert "Revert "drop unnecessary packages""

This reverts commit 161359ec4c690d66fcd780ae156c2d5910b1bce1.
---
 base_environment.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 52b3310..1ee3ebd 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -72,11 +72,9 @@ dependencies:
   - zarr
   - zeromq
   - pip:
-    - mapbox
     - py-noaa
     - sidecar
     - climate-toolbox
-    - impactlab-tools
     - parameterize-jobs
     - rhg_compute_tools
 name: base

From 25f27066cdb24e2d91801df9bac87a9ab9a0f59d Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:15 -0700
Subject: [PATCH 028/224] Revert "undo revert"

This reverts commit 3ed475aba3bf9470d0b01aec1c71529b34a2989e.
---
 base_environment.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/base_environment.yml b/base_environment.yml
index 1ee3ebd..52b3310 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -72,9 +72,11 @@ dependencies:
   - zarr
   - zeromq
   - pip:
+    - mapbox
     - py-noaa
     - sidecar
     - climate-toolbox
+    - impactlab-tools
     - parameterize-jobs
     - rhg_compute_tools
 name: base

From fecb4879fe2831d8fdbee2f655a0c34501fe0b9e Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:16 -0700
Subject: [PATCH 029/224] Revert "Revert "drop unnecessary packages""

This reverts commit aa1f1d30307257e134a994e0e0cbb1810cd0db26.
---
 base_environment.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 52b3310..1ee3ebd 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -72,11 +72,9 @@ dependencies:
   - zarr
   - zeromq
   - pip:
-    - mapbox
     - py-noaa
     - sidecar
     - climate-toolbox
-    - impactlab-tools
     - parameterize-jobs
     - rhg_compute_tools
 name: base

From 3b6f4d28ceba317f44052a87a3fac5e923a61a0b Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:16 -0700
Subject: [PATCH 030/224] Revert "remove packages added since last commit where
 solver worked"

This reverts commit cd2cb1dace4a84038a585a0fd85610b34ad46c5d.
---
 base_environment.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 1ee3ebd..36fd320 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -2,11 +2,11 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4
-  # - blas
+  - blas
   - bottleneck
   - boto3
   - bqplot
-  # - bump2version
+  - bump2version
   - cachecontrol
   - cartopy
   - cftime
@@ -46,7 +46,7 @@ dependencies:
   - numba
   - pandas
   - phantomjs # optional dependency of geoviews
-  # - pillow  # optional dependency of geoviews
+  - pillow  # optional dependency of geoviews
   - pip
   - plotly
   - pyinterp

From 125d288885d4c80a9201e8dc4dc5c6930a8ff306 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:17 -0700
Subject: [PATCH 031/224] Revert "tabs to match earlier commit"

This reverts commit 1d94e49ce77c7bc2d10fcad5fdda01b571064041.
---
 base_environment.yml | 154 +++++++++++++++++++++----------------------
 1 file changed, 77 insertions(+), 77 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 36fd320..7bae900 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,80 +1,80 @@
 channels:
-  - conda-forge
+- conda-forge
 dependencies:
-  - beautifulsoup4
-  - blas
-  - bottleneck
-  - boto3
-  - bqplot
-  - bump2version
-  - cachecontrol
-  - cartopy
-  - cftime
-  - click
-  - compilers
-  - dask
-  - dask-glm
-  - dask-kubernetes
-  - dask-ml
-  - datashader
-  - distributed
-  - dropbox
-  - esmpy
-  - fastparquet
-  - fiona
-  - gcsfs
-  - gdal
-  - geopandas
-  - geotiff
-  - geoviews
-  - git
-  - gitpython
-  - google-cloud-container
-  - google-cloud-storage
-  - holoviews
-  - h5netcdf
-  - intake-esm
-  - jedi
-  - jupyter-server-proxy
-  - kubernetes
-  - lz4
-  - make
-  - matplotlib
-  - nc-time-axis
-  - netcdf-fortran
-  - netcdf4
-  - numba
-  - pandas
-  - phantomjs # optional dependency of geoviews
-  - pillow  # optional dependency of geoviews
-  - pip
-  - plotly
-  - pyinterp
-  - pyshp
-  - python
-  - pyyaml
-  - rasterio
-  - regionmask
-  - rtree
-  - scikit-image
-  - scikit-learn
-  - scipy
-  - seaborn
-  - selenium # optional dependency of geoviews
-  - shapely
-  - sparse
-  - statsmodels
-  - tini
-  - xarray
-  - xesmf
-  - xhistogram
-  - xlrd
-  - zarr
-  - zeromq
-  - pip:
-    - py-noaa
-    - sidecar
-    - climate-toolbox
-    - parameterize-jobs
-    - rhg_compute_tools
+- beautifulsoup4
+- blas
+- bottleneck
+- boto3
+- bqplot
+- bump2version
+- cachecontrol
+- cartopy
+- cftime
+- click
+- compilers
+- dask
+- dask-glm
+- dask-kubernetes
+- dask-ml
+- datashader
+- distributed
+- dropbox
+- esmpy
+- fastparquet
+- fiona
+- gcsfs
+- gdal
+- geopandas
+- geotiff
+- geoviews
+- git
+- gitpython
+- google-cloud-container
+- google-cloud-storage
+- holoviews
+- h5netcdf
+- intake-esm
+- jedi
+- jupyter-server-proxy
+- kubernetes
+- lz4
+- make
+- matplotlib
+- nc-time-axis
+- netcdf-fortran
+- netcdf4
+- numba
+- pandas
+- phantomjs # optional dependency of geoviews
+- pillow  # optional dependency of geoviews
+- pip
+- plotly
+- pyinterp
+- pyshp
+- python
+- pyyaml
+- rasterio
+- regionmask
+- rtree
+- scikit-image
+- scikit-learn
+- scipy
+- seaborn
+- selenium # optional dependency of geoviews
+- shapely
+- sparse
+- statsmodels
+- tini
+- xarray
+- xesmf
+- xhistogram
+- xlrd
+- zarr
+- zeromq
+- pip:
+  - py-noaa
+  - sidecar
+  - climate-toolbox
+  - parameterize-jobs
+  - rhg_compute_tools
 name: base

From 5bf4ebd690c225d7243d0f5e14a2a4dc139aa751 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:17 -0700
Subject: [PATCH 032/224] Revert "unpin everything"

This reverts commit ed3e96380834567ff8520bcebab402342480bec2.
---
 base_environment.yml | 134 +++++++++++++++++++++----------------------
 1 file changed, 67 insertions(+), 67 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 7bae900..948f648 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,76 +1,76 @@
 channels:
 - conda-forge
 dependencies:
-- beautifulsoup4
+- beautifulsoup4=4.9
 - blas
-- bottleneck
-- boto3
-- bqplot
+- bottleneck=1.3
+- boto3=1.13
+- bqplot=0.12
 - bump2version
-- cachecontrol
-- cartopy
-- cftime
-- click
-- compilers
-- dask
-- dask-glm
-- dask-kubernetes
-- dask-ml
-- datashader
-- distributed
-- dropbox
-- esmpy
-- fastparquet
-- fiona
-- gcsfs
-- gdal
-- geopandas
-- geotiff
-- geoviews
-- git
-- gitpython
-- google-cloud-container
-- google-cloud-storage
-- holoviews
-- h5netcdf
-- intake-esm
-- jedi
-- jupyter-server-proxy
-- kubernetes
-- lz4
-- make
-- matplotlib
-- nc-time-axis
-- netcdf-fortran
-- netcdf4
-- numba
-- pandas
-- phantomjs # optional dependency of geoviews
+- cachecontrol=0.12
+- cartopy=0.17
+- cftime=1.1
+- click=7.1
+- compilers=1.0
+- dask=2.17
+- dask-glm=0.2
+- dask-kubernetes=0.10
+- dask-ml=1.5
+- datashader=0.11
+- distributed=2.17
+- dropbox=9.4
+- esmpy=7.1
+- fastparquet=0.4
+- fiona=1.8
+- gcsfs=0.6
+- gdal=3.0
+- geopandas=0.7
+- geotiff=1.5
+- geoviews=1.8
+- git=2.26
+- gitpython=3.1
+- google-cloud-container=0.3
+- google-cloud-storage=1.28
+- holoviews=1.13
+- h5netcdf=0.8
+- intake-esm=2020.5
+- jedi=0.17
+- jupyter-server-proxy=1.5
+- kubernetes=1.18
+- lz4=3.0
+- make=4.3
+- matplotlib=3.1
+- nc-time-axis=1.2
+- netcdf-fortran=4.4
+- netcdf4=1.5
+- numba=0.48
+- pandas=1.0
+- phantomjs=2.1 # optional dependency of geoviews
 - pillow  # optional dependency of geoviews
-- pip
-- plotly
-- pyinterp
-- pyshp
-- python
-- pyyaml
-- rasterio
-- regionmask
-- rtree
-- scikit-image
-- scikit-learn
-- scipy
-- seaborn
-- selenium # optional dependency of geoviews
-- shapely
-- sparse
-- statsmodels
-- tini
-- xarray
-- xesmf
-- xhistogram
-- xlrd
-- zarr
-- zeromq
+- pip=19.3
+- plotly=4.8
+- pyinterp=0.3
+- pyshp=2.1
+- python=3.7
+- pyyaml=5.3
+- rasterio=1.1
+- regionmask=0.5
+- rtree=0.9
+- scikit-image=0.17
+- scikit-learn=0.23
+- scipy=1.4
+- seaborn=0.10
+- selenium=3.141 # optional dependency of geoviews
+- shapely=1.7
+- sparse=0.9
+- statsmodels=0.11
+- tini=0.18
+- xarray=0.15
+- xesmf=0.3
+- xhistogram=0.1
+- xlrd=1.2
+- zarr=2.4
+- zeromq=4.3
 - pip:
   - py-noaa
   - sidecar

From 6b0e7d8a1e303a8d5862113b604ed66fd2772930 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:17 -0700
Subject: [PATCH 033/224] Revert "Revert "add climada""

This reverts commit efef32750300072988ae33f6afbaa7327953174c.
---
 base_environment.yml | 136 +++++++++++++++++++++----------------------
 common.sh            |   7 +++
 2 files changed, 75 insertions(+), 68 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 948f648..beef353 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,76 +1,76 @@
 channels:
 - conda-forge
 dependencies:
-- beautifulsoup4=4.9
+- beautifulsoup4
 - blas
-- bottleneck=1.3
-- boto3=1.13
-- bqplot=0.12
+- bottleneck
+- boto3
+- bqplot
 - bump2version
-- cachecontrol=0.12
-- cartopy=0.17
-- cftime=1.1
-- click=7.1
-- compilers=1.0
-- dask=2.17
-- dask-glm=0.2
-- dask-kubernetes=0.10
-- dask-ml=1.5
-- datashader=0.11
-- distributed=2.17
-- dropbox=9.4
-- esmpy=7.1
-- fastparquet=0.4
-- fiona=1.8
-- gcsfs=0.6
-- gdal=3.0
-- geopandas=0.7
-- geotiff=1.5
-- geoviews=1.8
-- git=2.26
-- gitpython=3.1
-- google-cloud-container=0.3
-- google-cloud-storage=1.28
-- holoviews=1.13
-- h5netcdf=0.8
-- intake-esm=2020.5
-- jedi=0.17
-- jupyter-server-proxy=1.5
-- kubernetes=1.18
-- lz4=3.0
-- make=4.3
-- matplotlib=3.1
-- nc-time-axis=1.2
-- netcdf-fortran=4.4
-- netcdf4=1.5
-- numba=0.48
-- pandas=1.0
-- phantomjs=2.1 # optional dependency of geoviews
-- pillow  # optional dependency of geoviews
-- pip=19.3
-- plotly=4.8
-- pyinterp=0.3
-- pyshp=2.1
-- python=3.7
-- pyyaml=5.3
-- rasterio=1.1
-- regionmask=0.5
-- rtree=0.9
-- scikit-image=0.17
-- scikit-learn=0.23
-- scipy=1.4
-- seaborn=0.10
-- selenium=3.141 # optional dependency of geoviews
-- shapely=1.7
-- sparse=0.9
-- statsmodels=0.11
-- tini=0.18
-- xarray=0.15
-- xesmf=0.3
-- xhistogram=0.1
-- xlrd=1.2
-- zarr=2.4
-- zeromq=4.3
+- cachecontrol
+- cartopy
+- cftime
+- click
+- compilers
+- dask
+- dask-glm
+- dask-kubernetes
+- dask-ml
+- datashader
+- distributed
+- dropbox
+- esmpy
+- fastparquet
+- fiona
+- gcsfs
+- gdal
+- geopandas
+- geotiff
+- geoviews
+- git
+- gitpython
+- google-cloud-container
+- google-cloud-storage
+- holoviews
+- h5netcdf
+- intake-esm
+- jedi
+- jupyter-server-proxy
+- kubernetes
+- lz4
+- make
+- matplotlib
+- nc-time-axis
+- netcdf-fortran
+- netcdf4
+- numba
+- pandas
+- phantomjs # optional dependency of geoviews
+- pillow # optional dependency of geoviews
+- pip
+- plotly
+- pyinterp
+- pyshp
+- python
+- pyyaml
+- rasterio
+- regionmask
+- rtree
+- scikit-image
+- scikit-learn
+- scipy
+- seaborn
+- selenium # optional dependency of geoviews
+- shapely
+- sparse
+- statsmodels
+- tini
+- xarray
+- xesmf
+- xhistogram
+- xlrd
+- zarr
+- zeromq
 - pip:
   - py-noaa
   - sidecar
diff --git a/common.sh b/common.sh
index f54e9a6..e47273c 100644
--- a/common.sh
+++ b/common.sh
@@ -43,6 +43,13 @@ unzip -o /usr/bin/vdatum_EGM1996.zip
 rm /usr/bin/vdatum.zip
 rm /usr/bin/vdatum_EGM1996.zip
 
+# download climada
+wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
+unzip /usr/bin/climada_python-1.4.2.zip
+rm /usr/bin/climada_python-1.4.2.zip
+sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
+sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
+
 # filepath curating
 chmod +x /usr/bin/prepare.sh
 mkdir /gcs

From 49b5384324c3f5abe04b8f560c581c902a974ebb Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:17 -0700
Subject: [PATCH 034/224] Revert "Revert "fix paths""

This reverts commit 33b41b5f5bd6b9e62df1891239e68ef91860eb5a.
---
 common.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common.sh b/common.sh
index e47273c..10ea39f 100644
--- a/common.sh
+++ b/common.sh
@@ -47,8 +47,8 @@ rm /usr/bin/vdatum_EGM1996.zip
 wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
 unzip /usr/bin/climada_python-1.4.2.zip
 rm /usr/bin/climada_python-1.4.2.zip
-sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
-sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
+sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
+sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
 
 # filepath curating
 chmod +x /usr/bin/prepare.sh

From 301276200f94fa5c953b57333dc126508bb63f55 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:18 -0700
Subject: [PATCH 035/224] Revert "Revert "try separate env instead""

This reverts commit 1052a4056c725f648ff3738fc02dcc4574be88df.
---
 common.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/common.sh b/common.sh
index 10ea39f..418be39 100644
--- a/common.sh
+++ b/common.sh
@@ -47,8 +47,7 @@ rm /usr/bin/vdatum_EGM1996.zip
 wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
 unzip /usr/bin/climada_python-1.4.2.zip
 rm /usr/bin/climada_python-1.4.2.zip
-sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
-sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
+conda env create -f /usr/bin/climada_python-1.4.2/requirements/env_climada.yml --name climada_env
 
 # filepath curating
 chmod +x /usr/bin/prepare.sh

From 26bb5c4680e22fc5cf93e67250830b94278a4fa0 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:18 -0700
Subject: [PATCH 036/224] Revert "try separate env instead"

This reverts commit cf43aa6dd3cd00daf7618ba4f1fc3e1bfc55ef1a.
---
 common.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/common.sh b/common.sh
index 418be39..10ea39f 100644
--- a/common.sh
+++ b/common.sh
@@ -47,7 +47,8 @@ rm /usr/bin/vdatum_EGM1996.zip
 wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
 unzip /usr/bin/climada_python-1.4.2.zip
 rm /usr/bin/climada_python-1.4.2.zip
-conda env create -f /usr/bin/climada_python-1.4.2/requirements/env_climada.yml --name climada_env
+sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
+sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
 
 # filepath curating
 chmod +x /usr/bin/prepare.sh

From aa2ba39291daf3f8df2824703daa267f74c158f6 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:18 -0700
Subject: [PATCH 037/224] Revert "fix paths"

This reverts commit 2dea811a821b2a8650d1c0979fffba181adaa6af.
---
 common.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common.sh b/common.sh
index 10ea39f..e47273c 100644
--- a/common.sh
+++ b/common.sh
@@ -47,8 +47,8 @@ rm /usr/bin/vdatum_EGM1996.zip
 wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
 unzip /usr/bin/climada_python-1.4.2.zip
 rm /usr/bin/climada_python-1.4.2.zip
-sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
-sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/requirements/env_climada.yml
+sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
+sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
 
 # filepath curating
 chmod +x /usr/bin/prepare.sh

From 3edd55d20fafae3fd6e00a81a274e91c3b664725 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:18 -0700
Subject: [PATCH 038/224] Revert "add climada"

This reverts commit 1d581250be59f248786394c15723e474c765c21f.
---
 base_environment.yml | 136 +++++++++++++++++++++----------------------
 common.sh            |   7 ---
 2 files changed, 68 insertions(+), 75 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index beef353..948f648 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,76 +1,76 @@
 channels:
 - conda-forge
 dependencies:
-- beautifulsoup4
+- beautifulsoup4=4.9
 - blas
-- bottleneck
-- boto3
-- bqplot
+- bottleneck=1.3
+- boto3=1.13
+- bqplot=0.12
 - bump2version
-- cachecontrol
-- cartopy
-- cftime
-- click
-- compilers
-- dask
-- dask-glm
-- dask-kubernetes
-- dask-ml
-- datashader
-- distributed
-- dropbox
-- esmpy
-- fastparquet
-- fiona
-- gcsfs
-- gdal
-- geopandas
-- geotiff
-- geoviews
-- git
-- gitpython
-- google-cloud-container
-- google-cloud-storage
-- holoviews
-- h5netcdf
-- intake-esm
-- jedi
-- jupyter-server-proxy
-- kubernetes
-- lz4
-- make
-- matplotlib
-- nc-time-axis
-- netcdf-fortran
-- netcdf4
-- numba
-- pandas
-- phantomjs # optional dependency of geoviews
-- pillow # optional dependency of geoviews
-- pip
-- plotly
-- pyinterp
-- pyshp
-- python
-- pyyaml
-- rasterio
-- regionmask
-- rtree
-- scikit-image
-- scikit-learn
-- scipy
-- seaborn
-- selenium # optional dependency of geoviews
-- shapely
-- sparse
-- statsmodels
-- tini
-- xarray
-- xesmf
-- xhistogram
-- xlrd
-- zarr
-- zeromq
+- cachecontrol=0.12
+- cartopy=0.17
+- cftime=1.1
+- click=7.1
+- compilers=1.0
+- dask=2.17
+- dask-glm=0.2
+- dask-kubernetes=0.10
+- dask-ml=1.5
+- datashader=0.11
+- distributed=2.17
+- dropbox=9.4
+- esmpy=7.1
+- fastparquet=0.4
+- fiona=1.8
+- gcsfs=0.6
+- gdal=3.0
+- geopandas=0.7
+- geotiff=1.5
+- geoviews=1.8
+- git=2.26
+- gitpython=3.1
+- google-cloud-container=0.3
+- google-cloud-storage=1.28
+- holoviews=1.13
+- h5netcdf=0.8
+- intake-esm=2020.5
+- jedi=0.17
+- jupyter-server-proxy=1.5
+- kubernetes=1.18
+- lz4=3.0
+- make=4.3
+- matplotlib=3.1
+- nc-time-axis=1.2
+- netcdf-fortran=4.4
+- netcdf4=1.5
+- numba=0.48
+- pandas=1.0
+- phantomjs=2.1 # optional dependency of geoviews
+- pillow  # optional dependency of geoviews
+- pip=19.3
+- plotly=4.8
+- pyinterp=0.3
+- pyshp=2.1
+- python=3.7
+- pyyaml=5.3
+- rasterio=1.1
+- regionmask=0.5
+- rtree=0.9
+- scikit-image=0.17
+- scikit-learn=0.23
+- scipy=1.4
+- seaborn=0.10
+- selenium=3.141 # optional dependency of geoviews
+- shapely=1.7
+- sparse=0.9
+- statsmodels=0.11
+- tini=0.18
+- xarray=0.15
+- xesmf=0.3
+- xhistogram=0.1
+- xlrd=1.2
+- zarr=2.4
+- zeromq=4.3
 - pip:
   - py-noaa
   - sidecar
diff --git a/common.sh b/common.sh
index e47273c..f54e9a6 100644
--- a/common.sh
+++ b/common.sh
@@ -43,13 +43,6 @@ unzip -o /usr/bin/vdatum_EGM1996.zip
 rm /usr/bin/vdatum.zip
 rm /usr/bin/vdatum_EGM1996.zip
 
-# download climada
-wget https://github.com/CLIMADA-project/climada_python/archive/v1.4.2.zip -O /usr/bin/climada_python-1.4.2.zip
-unzip /usr/bin/climada_python-1.4.2.zip
-rm /usr/bin/climada_python-1.4.2.zip
-sed -i 's/==/=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
-sed -i 's/=/>=/g' /usr/bin/climada_python-1.4.2/env_climada.yml
-
 # filepath curating
 chmod +x /usr/bin/prepare.sh
 mkdir /gcs

From 50b4b8344601c429a1c2bac42a32043255652fb9 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:19 -0700
Subject: [PATCH 039/224] Revert "more drops"

This reverts commit dfe0cbb3ccd0bb6a85c1f42f33887e3077a17b26.
---
 base_environment.yml | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 948f648..e2d740c 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -2,7 +2,6 @@ channels:
 - conda-forge
 dependencies:
 - beautifulsoup4=4.9
-- blas
 - bottleneck=1.3
 - boto3=1.13
 - bqplot=0.12
@@ -25,6 +24,7 @@ dependencies:
 - gcsfs=0.6
 - gdal=3.0
 - geopandas=0.7
+- geopy=1.22
 - geotiff=1.5
 - geoviews=1.8
 - git=2.26
@@ -33,25 +33,34 @@ dependencies:
 - google-cloud-storage=1.28
 - holoviews=1.13
 - h5netcdf=0.8
+- icu=58.2
 - intake-esm=2020.5
+- iris=2.4
 - jedi=0.17
 - jupyter-server-proxy=1.5
 - kubernetes=1.18
+- lapack=3.6
 - lz4=3.0
 - make=4.3
 - matplotlib=3.1
 - nc-time-axis=1.2
+- ncurses=6.1
 - netcdf-fortran=4.4
 - netcdf4=1.5
 - numba=0.48
+- numcodecs=0.6
 - pandas=1.0
-- phantomjs=2.1 # optional dependency of geoviews
-- pillow  # optional dependency of geoviews
+- phantomjs=2.1
 - pip=19.3
 - plotly=4.8
+- polyline=1.4
+- pygeos=0.6
 - pyinterp=0.3
 - pyshp=2.1
 - python=3.7
+- python-blosc=1.9
+- python-snappy=0.5
+- pyviz_comms=0.7
 - pyyaml=5.3
 - rasterio=1.1
 - regionmask=0.5
@@ -60,7 +69,7 @@ dependencies:
 - scikit-learn=0.23
 - scipy=1.4
 - seaborn=0.10
-- selenium=3.141 # optional dependency of geoviews
+- selenium=3.141
 - shapely=1.7
 - sparse=0.9
 - statsmodels=0.11

From 2696670787ff284e042628b94dd968f8ae6ab44f Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:19 -0700
Subject: [PATCH 040/224] Revert "drop fusepy"

This reverts commit e6c83cd19a29ad539330a122844a5f1964d9a7a6.
---
 base_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/base_environment.yml b/base_environment.yml
index e2d740c..a5f6a97 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -21,6 +21,7 @@ dependencies:
 - esmpy=7.1
 - fastparquet=0.4
 - fiona=1.8
+- fusepy=3.0
 - gcsfs=0.6
 - gdal=3.0
 - geopandas=0.7

From a135f99aa5fa9972fc1dc38677aa6128755b2c05 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:19 -0700
Subject: [PATCH 041/224] Revert "drop some packages"

This reverts commit 104822f9b2b3c5e8069a8b731cedc95f04c4ed00.
---
 base_environment.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/base_environment.yml b/base_environment.yml
index a5f6a97..36b5a06 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -5,7 +5,7 @@ dependencies:
 - bottleneck=1.3
 - boto3=1.13
 - bqplot=0.12
-- bump2version
+- bumpversion=0.5
 - cachecontrol=0.12
 - cartopy=0.17
 - cftime=1.1
@@ -16,14 +16,17 @@ dependencies:
 - dask-kubernetes=0.10
 - dask-ml=1.5
 - datashader=0.11
+- descartes=1.1
 - distributed=2.17
 - dropbox=9.4
 - esmpy=7.1
 - fastparquet=0.4
 - fiona=1.8
 - fusepy=3.0
+- gcc_linux-64=7.5
 - gcsfs=0.6
 - gdal=3.0
+- geoalchemy2=0.6
 - geopandas=0.7
 - geopy=1.22
 - geotiff=1.5
@@ -75,12 +78,17 @@ dependencies:
 - sparse=0.9
 - statsmodels=0.11
 - tini=0.18
+- unzip=6.0
+- uritemplate=3.0
 - xarray=0.15
 - xesmf=0.3
+- xgcm=0.3
 - xhistogram=0.1
 - xlrd=1.2
+- xrft=0.2
 - zarr=2.4
 - zeromq=4.3
+- zict=2.0
 - pip:
   - py-noaa
   - sidecar

From 363600f7c6f53f5977cdda037806b67bd585de22 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:03:19 -0700
Subject: [PATCH 042/224] Revert "use minor versions from resolved conda travis
 output"

This reverts commit dd20e0e397474b32dfb10641b2c9772e0244da90.
---
 base_environment.yml | 201 ++++++++++++++++++++++---------------------
 1 file changed, 105 insertions(+), 96 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 36b5a06..b3a05cd 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,98 +1,107 @@
+name: base
 channels:
-- conda-forge
+  - conda-forge
 dependencies:
-- beautifulsoup4=4.9
-- bottleneck=1.3
-- boto3=1.13
-- bqplot=0.12
-- bumpversion=0.5
-- cachecontrol=0.12
-- cartopy=0.17
-- cftime=1.1
-- click=7.1
-- compilers=1.0
-- dask=2.17
-- dask-glm=0.2
-- dask-kubernetes=0.10
-- dask-ml=1.5
-- datashader=0.11
-- descartes=1.1
-- distributed=2.17
-- dropbox=9.4
-- esmpy=7.1
-- fastparquet=0.4
-- fiona=1.8
-- fusepy=3.0
-- gcc_linux-64=7.5
-- gcsfs=0.6
-- gdal=3.0
-- geoalchemy2=0.6
-- geopandas=0.7
-- geopy=1.22
-- geotiff=1.5
-- geoviews=1.8
-- git=2.26
-- gitpython=3.1
-- google-cloud-container=0.3
-- google-cloud-storage=1.28
-- holoviews=1.13
-- h5netcdf=0.8
-- icu=58.2
-- intake-esm=2020.5
-- iris=2.4
-- jedi=0.17
-- jupyter-server-proxy=1.5
-- kubernetes=1.18
-- lapack=3.6
-- lz4=3.0
-- make=4.3
-- matplotlib=3.1
-- nc-time-axis=1.2
-- ncurses=6.1
-- netcdf-fortran=4.4
-- netcdf4=1.5
-- numba=0.48
-- numcodecs=0.6
-- pandas=1.0
-- phantomjs=2.1
-- pip=19.3
-- plotly=4.8
-- polyline=1.4
-- pygeos=0.6
-- pyinterp=0.3
-- pyshp=2.1
-- python=3.7
-- python-blosc=1.9
-- python-snappy=0.5
-- pyviz_comms=0.7
-- pyyaml=5.3
-- rasterio=1.1
-- regionmask=0.5
-- rtree=0.9
-- scikit-image=0.17
-- scikit-learn=0.23
-- scipy=1.4
-- seaborn=0.10
-- selenium=3.141
-- shapely=1.7
-- sparse=0.9
-- statsmodels=0.11
-- tini=0.18
-- unzip=6.0
-- uritemplate=3.0
-- xarray=0.15
-- xesmf=0.3
-- xgcm=0.3
-- xhistogram=0.1
-- xlrd=1.2
-- xrft=0.2
-- zarr=2.4
-- zeromq=4.3
-- zict=2.0
-- pip:
-  - py-noaa
-  - sidecar
-  - climate-toolbox
-  - parameterize-jobs
-  - rhg_compute_tools
-name: base
+  - beautifulsoup4
+  - bottleneck
+  - boto3
+  - bqplot
+  - bumpversion
+  - cachecontrol
+  - cartopy
+  - cftime
+  - click
+  - compilers
+  - dask
+  - dask-glm
+  - dask-kubernetes
+  - dask-ml
+  - datashader
+  - descartes
+  - distributed
+  - dropbox
+# need to make sure we get esmpy compiled with mpi otherwise xesmf regridding
+# won't work
+  - esmpy
+  - fastparquet
+  - fiona
+  - fusepy
+# this gcc pin is necessary b/c of a weird feature in the h553295d_15 build
+# which makes it hard to build numpy-based cython extensions (like pyclaw).
+# we should try removing it whenever we next do an update and see if Clawpack
+# can still be built
+  - gcc_linux-64
+  - gcsfs
+  - gdal
+  - geoalchemy2
+  - geopandas
+  - geopy
+  - geotiff
+  - geoviews
+  - git
+  - gitpython
+  - google-cloud-container
+  - google-cloud-storage
+  - holoviews
+  - h5netcdf
+  - icu
+  - intake-esm
+  - iris
+  - jedi
+# need server proxy on workers if using remote scheduler
+  - jupyter-server-proxy
+  - kubernetes
+  - lapack
+  - lz4
+  - make
+  - matplotlib
+  - nc-time-axis
+  - ncurses
+  - netcdf-fortran
+  - netcdf4
+  - numba
+  - numcodecs
+  - pandas
+  # for geoviews
+  - phantomjs
+  - pip
+  - plotly
+  - polyline
+  - pygeos
+  - pyinterp
+  - pyshp
+  - python
+  - python-blosc
+  - python-snappy
+  - pyviz_comms
+  - pyyaml
+  - rasterio
+  - regionmask
+  - rtree
+  - scikit-image
+  - scikit-learn
+  - scipy
+  - seaborn
+  # for geoviews
+  - selenium
+  - shapely
+  - sparse
+  - statsmodels
+  - tini
+  - unzip
+  - uritemplate
+  - xarray
+  - xesmf
+  - xgcm
+  - xhistogram
+  - xlrd
+  - xrft
+  - zarr
+  - zeromq
+  - zict
+  - pip:
+    - py-noaa
+    - sidecar
+    - climate-toolbox
+    - parameterize-jobs
+    - rhg_compute_tools

From 596446584ade62612e3ca99b78745dcb39eaa5c8 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:51:07 -0700
Subject: [PATCH 043/224] replace bumpversion

---
 base_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base_environment.yml b/base_environment.yml
index b3a05cd..6901f32 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -6,7 +6,7 @@ dependencies:
   - bottleneck
   - boto3
   - bqplot
-  - bumpversion
+  - bump2version
   - cachecontrol
   - cartopy
   - cftime

From a6ba44ae1b4a7df69f72ef4c534f1e1908937cfc Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:52:39 -0700
Subject: [PATCH 044/224] drop some packages

---
 base_environment.yml | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 6901f32..efbf0ff 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -17,7 +17,6 @@ dependencies:
   - dask-kubernetes
   - dask-ml
   - datashader
-  - descartes
   - distributed
   - dropbox
 # need to make sure we get esmpy compiled with mpi otherwise xesmf regridding
@@ -26,14 +25,8 @@ dependencies:
   - fastparquet
   - fiona
   - fusepy
-# this gcc pin is necessary b/c of a weird feature in the h553295d_15 build
-# which makes it hard to build numpy-based cython extensions (like pyclaw).
-# we should try removing it whenever we next do an update and see if Clawpack
-# can still be built
-  - gcc_linux-64
   - gcsfs
   - gdal
-  - geoalchemy2
   - geopandas
   - geopy
   - geotiff
@@ -88,17 +81,12 @@ dependencies:
   - sparse
   - statsmodels
   - tini
-  - unzip
-  - uritemplate
   - xarray
   - xesmf
-  - xgcm
   - xhistogram
   - xlrd
-  - xrft
   - zarr
   - zeromq
-  - zict
   - pip:
     - py-noaa
     - sidecar

From 295fa5f8e852ca4cd22561fadb64bd1938c753a1 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:53:19 -0700
Subject: [PATCH 045/224] drop fusepy

---
 base_environment.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/base_environment.yml b/base_environment.yml
index efbf0ff..921f188 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -24,7 +24,6 @@ dependencies:
   - esmpy
   - fastparquet
   - fiona
-  - fusepy
   - gcsfs
   - gdal
   - geopandas

From 07b03b134f3bef13d7877b45dfb71288509c673b Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:55:31 -0700
Subject: [PATCH 046/224] drop more packages

---
 base_environment.yml | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 921f188..3afc135 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -27,7 +27,6 @@ dependencies:
   - gcsfs
   - gdal
   - geopandas
-  - geopy
   - geotiff
   - geoviews
   - git
@@ -36,9 +35,7 @@ dependencies:
   - google-cloud-storage
   - holoviews
   - h5netcdf
-  - icu
   - intake-esm
-  - iris
   - jedi
 # need server proxy on workers if using remote scheduler
   - jupyter-server-proxy
@@ -48,24 +45,17 @@ dependencies:
   - make
   - matplotlib
   - nc-time-axis
-  - ncurses
   - netcdf-fortran
   - netcdf4
   - numba
-  - numcodecs
   - pandas
   # for geoviews
   - phantomjs
   - pip
   - plotly
-  - polyline
-  - pygeos
   - pyinterp
   - pyshp
   - python
-  - python-blosc
-  - python-snappy
-  - pyviz_comms
   - pyyaml
   - rasterio
   - regionmask

From 3c57f4e809aeb69c1432b40769349e9482555075 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:55:55 -0700
Subject: [PATCH 047/224] use blas

---
 base_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base_environment.yml b/base_environment.yml
index 3afc135..8baf15e 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -3,6 +3,7 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4
+  - blas
   - bottleneck
   - boto3
   - bqplot
@@ -40,7 +41,6 @@ dependencies:
 # need server proxy on workers if using remote scheduler
   - jupyter-server-proxy
   - kubernetes
-  - lapack
   - lz4
   - make
   - matplotlib

From 7079d07845ad10949cb2ce0168b4bf56bd73ccfb Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 13:56:18 -0700
Subject: [PATCH 048/224] use blas

---
 base_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/base_environment.yml b/base_environment.yml
index 8baf15e..600486b 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -51,6 +51,7 @@ dependencies:
   - pandas
   # for geoviews
   - phantomjs
+  - pillow
   - pip
   - plotly
   - pyinterp

From cce78d0f73b8253206f7c8da07b17c4de71c773c Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Fri, 12 Jun 2020 14:34:19 -0700
Subject: [PATCH 049/224] add fusepy

---
 base_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/base_environment.yml b/base_environment.yml
index 600486b..afb0e7c 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -25,6 +25,7 @@ dependencies:
   - esmpy
   - fastparquet
   - fiona
+  - fusepy
   - gcsfs
   - gdal
   - geopandas

From ba31bdd3954cb39c014c3cba928ee47aaf103a59 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Mon, 15 Jun 2020 13:17:36 -0700
Subject: [PATCH 050/224] update os, jupyter, miniconda

---
 notebook/Dockerfile | 2 +-
 worker/Dockerfile   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index aad9f6d..4a3fb72 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook:notebook-6.0.0
+FROM jupyter/base-notebook:notebook-6.0.3
 ARG DEBIAN_FRONTEND=noninteractive
 
 ## needed to make sure things with cython compile correctly
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 13ae443..988360a 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -1,5 +1,5 @@
 ## using same container as jupyter/base-notebook:python-3.7.3
-FROM ubuntu:bionic-20190612@sha256:6e9f67fa63b0323e9a1e587fd71c561ba48a034504fb804fd26fd8800039835d
+FROM ubuntu:bionic-20200403@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
 ARG DEBIAN_FRONTEND=noninteractive
 ENV NPY_DISTUTILS_APPEND_FLAGS=1
 
@@ -22,7 +22,7 @@ RUN bash /tempdir/common.sh
 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
 ENV PATH /opt/conda/bin:$PATH
 
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh -O ~/miniconda.sh && \
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.2-Linux-x86_64.sh -O ~/miniconda.sh && \
     /bin/bash ~/miniconda.sh -b -p /opt/conda && \
     rm ~/miniconda.sh && \
     ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \

From a1785a84527bbfceb5b7c226a84230422a3f635c Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Mon, 15 Jun 2020 14:16:16 -0700
Subject: [PATCH 051/224] remove dev conda channels

---
 notebook/Dockerfile | 3 +--
 worker/Dockerfile   | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 4a3fb72..3370d99 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -47,8 +47,7 @@ RUN mkdir /opt/conda/specs
 COPY base_environment.yml /opt/conda/specs
 COPY notebook_environment.yml /opt/conda/specs
 COPY r_environment.yml /opt/conda/specs
-RUN conda config --add channels conda-forge/label/dev && \
-  conda config --add channels conda-forge
+RUN conda config --add channels conda-forge
 
 
 ## set up conda
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 988360a..1abab13 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -34,8 +34,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.2-Linux
 ## set up conda channels
 RUN mkdir /opt/conda/specs
 COPY base_environment.yml /opt/conda/specs
-RUN conda config --add channels conda-forge/label/dev && \
-  conda config --add channels conda-forge
+RUN conda config --add channels conda-forge
 
 
 ## set up python env

From c27a0ef3e3f614f9eef67ae72bccbe447e7548b4 Mon Sep 17 00:00:00 2001
From: Daniel Allen <danielpaulallen@gmail.com>
Date: Tue, 16 Jun 2020 09:09:43 -0700
Subject: [PATCH 052/224] pin full build

---
 base_environment.yml | 163 +++++++++++++++++++++----------------------
 1 file changed, 79 insertions(+), 84 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index afb0e7c..833e041 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,86 +1,81 @@
-name: base
 channels:
-  - conda-forge
+- conda-forge
 dependencies:
-  - beautifulsoup4
-  - blas
-  - bottleneck
-  - boto3
-  - bqplot
-  - bump2version
-  - cachecontrol
-  - cartopy
-  - cftime
-  - click
-  - compilers
-  - dask
-  - dask-glm
-  - dask-kubernetes
-  - dask-ml
-  - datashader
-  - distributed
-  - dropbox
-# need to make sure we get esmpy compiled with mpi otherwise xesmf regridding
-# won't work
-  - esmpy
-  - fastparquet
-  - fiona
-  - fusepy
-  - gcsfs
-  - gdal
-  - geopandas
-  - geotiff
-  - geoviews
-  - git
-  - gitpython
-  - google-cloud-container
-  - google-cloud-storage
-  - holoviews
-  - h5netcdf
-  - intake-esm
-  - jedi
-# need server proxy on workers if using remote scheduler
-  - jupyter-server-proxy
-  - kubernetes
-  - lz4
-  - make
-  - matplotlib
-  - nc-time-axis
-  - netcdf-fortran
-  - netcdf4
-  - numba
-  - pandas
-  # for geoviews
-  - phantomjs
-  - pillow
-  - pip
-  - plotly
-  - pyinterp
-  - pyshp
-  - python
-  - pyyaml
-  - rasterio
-  - regionmask
-  - rtree
-  - scikit-image
-  - scikit-learn
-  - scipy
-  - seaborn
-  # for geoviews
-  - selenium
-  - shapely
-  - sparse
-  - statsmodels
-  - tini
-  - xarray
-  - xesmf
-  - xhistogram
-  - xlrd
-  - zarr
-  - zeromq
-  - pip:
-    - py-noaa
-    - sidecar
-    - climate-toolbox
-    - parameterize-jobs
-    - rhg_compute_tools
+- beautifulsoup4=4.9.1=py37hc8dfbb8_0
+- blas=2.15=mkl
+- bottleneck=1.3.2=py37h03ebfcd_1
+- boto3=1.14.3=pyh9f0ad1d_0
+- bqplot=0.12.12=pyh9f0ad1d_0
+- bump2version=1.0.0=pyh9f0ad1d_0
+- cachecontrol=0.12.6=py_0
+- cartopy=0.17.0=py37h423102d_1009
+- cftime=1.1.3=py37h03ebfcd_0
+- click=7.1.2=pyh9f0ad1d_0
+- compilers=1.0.4=0
+- dask=2.18.1=py_0
+- dask-glm=0.2.0=py_1
+- dask-kubernetes=0.10.1=py_0
+- dask-ml=1.5.0=py_0
+- datashader=0.11.0=pyh9f0ad1d_0
+- distributed=2.18.0=py37hc8dfbb8_0
+- dropbox=10.1.1=pyh9f0ad1d_0
+- esmpy=7.1.0=py37h5ca1d4c_3
+- fastparquet=0.4.0=py37h03ebfcd_0
+- fiona=1.8.13=py37h900e953_0
+- fusepy=3.0.1=py_0
+- gcsfs=0.6.2=py_0
+- gdal=3.0.2=py37hbb2a789_0
+- geopandas=0.7.0=py_1
+- geotiff=1.5.1=hbd99317_7
+- geoviews=1.8.1=py_0
+- git=2.26.0=pl526hf241897_0
+- gitpython=3.1.3=py_0
+- google-cloud-container=0.3.0=py37_0
+- google-cloud-storage=1.28.1=pyh9f0ad1d_0
+- holoviews=1.13.2=pyh9f0ad1d_0
+- h5netcdf=0.8.0=py_0
+- intake-esm=2020.6.11=py_0
+- jedi=0.17.0=py37hc8dfbb8_0
+- jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
+- kubernetes=1.18.3=0
+- lz4=3.0.2=py37hb076c26_1
+- make=4.3=h516909a_0
+- matplotlib=3.1.1=py37_0
+- nc-time-axis=1.2.0=py_1
+- netcdf-fortran=4.4.5=hea25ff8_1000
+- netcdf4=1.5.1.2=py37had58050_0
+- numba=0.48.0=py37hb3f55d8_0
+- pandas=1.0.4=py37h0da4684_0
+- phantomjs=2.1.1=1 # for geoviews
+- pillow=7.1.2=py37h718be6c_0
+- pip=20.1.1=py_1
+- plotly=4.8.1=pyh9f0ad1d_0
+- pyinterp=0.3.2=openblas_py37hcf60e08_2
+- pyshp=2.1.0=py_0
+- python=3.7.3=h5b0a415_0
+- pyyaml=5.3.1=py37h8f50634_0
+- rasterio=1.1.1=py37h900e953_0
+- regionmask=0.5.0=py_1
+- rtree=0.9.4=py37h8526d28_1
+- scikit-image=0.17.2=py37h0da4684_1
+- scikit-learn=0.23.1=py37h8a51577_0
+- scipy=1.4.1=py37ha3d9a3c_3
+- seaborn=0.10.1=1
+- selenium=3.141.0=py37h8f50634_1001 # for geoviews
+- shapely=1.7.0=py37hb106bac_1
+- sparse=0.9.1=py_0
+- statsmodels=0.11.1=py37h8f50634_2
+- tini=0.18.0=h14c3975_1001
+- xarray=0.15.1=py_0
+- xesmf=0.3.0=py_0
+- xhistogram=0.1.1=py_0
+- xlrd=1.2.0=py_0
+- zarr=2.4.0=py_0
+- zeromq=4.3.2=he1b5a44_2
+- pip:
+  - py-noaa
+  - sidecar
+  - climate-toolbox
+  - parameterize-jobs
+  - rhg_compute_tools
+name: base

From ec1c76db71c5bd6e4fa39a2249f05430357d97ee Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 10:28:17 -0700
Subject: [PATCH 053/224] drop compiler_compat hack

---
 common.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/common.sh b/common.sh
index f54e9a6..3d01f89 100644
--- a/common.sh
+++ b/common.sh
@@ -48,7 +48,3 @@ chmod +x /usr/bin/prepare.sh
 mkdir /gcs
 mkdir /opt/app
 
-# super sketchy hack to get around our need for compiler_compat binaries and some
-# other things that cause problems together?
-# see https://github.com/ContinuumIO/anaconda-issues/issues/11152
-rm -rf /opt/conda/compiler_compat/ld

From 3acda1c93f774685138dd6729dc713cc5a22d268 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 10:33:47 -0700
Subject: [PATCH 054/224] unpin notebook env

---
 notebook/notebook_environment.yml | 42 +++++++++++++++----------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 81d85f6..7ddc919 100644
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -2,26 +2,26 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - black=19.10b0=py37_0
-  - coverage=4.5.4=py37h516909a_0
-  - dask-labextension=1.0.3=py_0
-  - flake8=3.7.9=py37_0
-  - ipdb=0.12.3=py_0
-  - ipyleaflet=0.11.6=py37_0
-  - ipympl=0.3.3=py_0
-  - jupyterlab_code_formatter=0.7.0=py_0
-  - nano=2.9.8=hb256ff8_1000
-  - nb_conda_kernels=2.2.2=py37_0
-  - nose=1.3.7=py37_1003
-  - octave_kernel=0.31.0=py_0
-  - oct2py=5.0.4=py_0
-  - openssh=7.9p1=h0fa992c_1
-  - pip=19.3.1=py37_0
-  - pytest=5.3.1=py37_0
-  - pytest-cov=2.8.1=py_0
-  - pytest-runner=5.2=py_0
-  - python-graphviz=0.13.2=py_0
-  - sphinx=2.2.2=py_0
-  - tox=3.14.2=py_0
+  - black
+  - coverage
+  - dask-labextension
+  - flake8
+  - ipdb
+  - ipyleaflet
+  - ipympl
+  - isort
+  - jupyterlab_code_formatter
+  - nano
+  - nb_conda_kernels
+  - nose
+  - octave_kernel
+  - oct2py
+  - openssh
+  - pytest
+  - pytest-cov
+  - pytest-runner
+  - python-graphviz
+  - sphinx
+  - tox
   - pip:
     - black_nbconvert

From 7fb1e7b4a4fac754f3ed9d41a2538a72a6a85e79 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 10:44:11 -0700
Subject: [PATCH 055/224] add strict channel priority

---
 notebook/Dockerfile | 8 +++++---
 worker/Dockerfile   | 3 ++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 3370d99..aaad511 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -47,8 +47,8 @@ RUN mkdir /opt/conda/specs
 COPY base_environment.yml /opt/conda/specs
 COPY notebook_environment.yml /opt/conda/specs
 COPY r_environment.yml /opt/conda/specs
-RUN conda config --add channels conda-forge
-
+RUN conda config --add channels conda-forge && \
+  conda config --set channel_priority strict
 
 ## set up conda
 RUN conda update -n base conda
@@ -73,11 +73,13 @@ RUN jupyter labextension update --all && \
     @jupyter-widgets/jupyterlab-manager \
     @jupyter-widgets/jupyterlab-sidecar \
     @pyviz/jupyterlab_pyviz \
+    @ryantam626/jupyterlab_code_formatter \
     dask-labextension \
     jupyter-leaflet \
     jupyter-matplotlib \
     jupyterlab-plotly \
-    plotlywidget
+    plotlywidget && \
+    jupyter serverextension enable --py jupyterlab_code_formatter
 
 
 ## clean up
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 1abab13..fe6d7df 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -34,7 +34,8 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.2-Linux
 ## set up conda channels
 RUN mkdir /opt/conda/specs
 COPY base_environment.yml /opt/conda/specs
-RUN conda config --add channels conda-forge
+RUN conda config --add channels conda-forge && \
+    conda config --set channel_priority strict
 
 
 ## set up python env

From 2a15893228815285f02f57256cc09fb567836fb7 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 10:48:44 -0700
Subject: [PATCH 056/224] drop r_env

---
 notebook/Dockerfile        |  5 -----
 notebook/r_environment.yml | 24 ------------------------
 pin.py                     |  8 ++++----
 3 files changed, 4 insertions(+), 33 deletions(-)
 delete mode 100644 notebook/r_environment.yml

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index aaad511..3553ed0 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -46,7 +46,6 @@ RUN sudo apt-get install -yq \
 RUN mkdir /opt/conda/specs
 COPY base_environment.yml /opt/conda/specs
 COPY notebook_environment.yml /opt/conda/specs
-COPY r_environment.yml /opt/conda/specs
 RUN conda config --add channels conda-forge && \
   conda config --set channel_priority strict
 
@@ -61,10 +60,6 @@ RUN conda env update -f /opt/conda/specs/notebook_environment.yml
 
 RUN conda list -n base
 
-# create r environment
-RUN conda env create -f /opt/conda/specs/r_environment.yml
-
-RUN conda list -n r
 
 ## Set up jupyter lab extensions
 RUN jupyter labextension update --all && \
diff --git a/notebook/r_environment.yml b/notebook/r_environment.yml
deleted file mode 100644
index 09e3e50..0000000
--- a/notebook/r_environment.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-name: r
-channels:
-  - conda-forge
-  - r
-dependencies:
-  - rpy2=3.1.0=py38r36hc1659b7_3
-  - r-base=3.6.1=h3a67422_6
-  - r-irkernel=1.1=r36h6115d3f_0
-  - r-plyr=1.8.4=r36h0357c0b_1003
-  - r-devtools=2.2.1=r36h6115d3f_0
-  - r-tidyverse=1.3.0=r36h6115d3f_0
-  - r-shiny=1.4.0=r36h6115d3f_0
-  - r-rmarkdown=1.18=r36h6115d3f_0
-  - r-forecast=8.10=r36h0357c0b_0
-  - r-rsqlite=2.1.4=r36h0357c0b_0
-  - r-reshape2=1.4.3=r36h0357c0b_1004
-  - r-nycflights13=1.0.1=r36h6115d3f_0
-  - r-caret=6.0_84=r36hcdcec82_1
-  - r-rcurl=1.95_4.12=r36hcdcec82_1
-  - r-crayon=1.3.4=r36h6115d3f_1002
-  - r-randomforest=4.6_14=r36h9bbef5b_1002
-  - r-sparklyr=1.0.5=r36h6115d3f_0
-  - r-htmlwidgets=1.5.1=r36h6115d3f_0
-  - r-hexbin=1.28.0=r36h9bbef5b_0
diff --git a/pin.py b/pin.py
index 175ab22..591ddd2 100644
--- a/pin.py
+++ b/pin.py
@@ -297,8 +297,8 @@ def pin(file, dry_run):
     spec_files = [
             ('base_environment.yml', 'base'),
             ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base'),
-            ('notebook/r_environment.yml', 'r')]
+            ('octave-worker/octave_environment.yml', 'base')
+    ]
     
     if file == 'all':
         pin_files(spec_files, dry_run=dry_run)
@@ -332,8 +332,8 @@ def unpin(file, dry_run):
     spec_files = [
             ('base_environment.yml', 'base'),
             ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base'),
-            ('notebook/r_environment.yml', 'r')]
+            ('octave-worker/octave_environment.yml', 'base')
+    ]
     
     if file == 'all':
         unpin_files(spec_files, dry_run=dry_run)

From be715eee9f712cb85edbc3cbea45ee3ea64e004e Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 11:03:36 -0700
Subject: [PATCH 057/224] unpin

---
 base_environment.yml                 | 156 +++++++++++++--------------
 octave-worker/octave_environment.yml |   4 +-
 pin.py                               |   8 +-
 3 files changed, 84 insertions(+), 84 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 833e041..0aa73ea 100644
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,81 +1,81 @@
 channels:
-- conda-forge
+  - conda-forge
 dependencies:
-- beautifulsoup4=4.9.1=py37hc8dfbb8_0
-- blas=2.15=mkl
-- bottleneck=1.3.2=py37h03ebfcd_1
-- boto3=1.14.3=pyh9f0ad1d_0
-- bqplot=0.12.12=pyh9f0ad1d_0
-- bump2version=1.0.0=pyh9f0ad1d_0
-- cachecontrol=0.12.6=py_0
-- cartopy=0.17.0=py37h423102d_1009
-- cftime=1.1.3=py37h03ebfcd_0
-- click=7.1.2=pyh9f0ad1d_0
-- compilers=1.0.4=0
-- dask=2.18.1=py_0
-- dask-glm=0.2.0=py_1
-- dask-kubernetes=0.10.1=py_0
-- dask-ml=1.5.0=py_0
-- datashader=0.11.0=pyh9f0ad1d_0
-- distributed=2.18.0=py37hc8dfbb8_0
-- dropbox=10.1.1=pyh9f0ad1d_0
-- esmpy=7.1.0=py37h5ca1d4c_3
-- fastparquet=0.4.0=py37h03ebfcd_0
-- fiona=1.8.13=py37h900e953_0
-- fusepy=3.0.1=py_0
-- gcsfs=0.6.2=py_0
-- gdal=3.0.2=py37hbb2a789_0
-- geopandas=0.7.0=py_1
-- geotiff=1.5.1=hbd99317_7
-- geoviews=1.8.1=py_0
-- git=2.26.0=pl526hf241897_0
-- gitpython=3.1.3=py_0
-- google-cloud-container=0.3.0=py37_0
-- google-cloud-storage=1.28.1=pyh9f0ad1d_0
-- holoviews=1.13.2=pyh9f0ad1d_0
-- h5netcdf=0.8.0=py_0
-- intake-esm=2020.6.11=py_0
-- jedi=0.17.0=py37hc8dfbb8_0
-- jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
-- kubernetes=1.18.3=0
-- lz4=3.0.2=py37hb076c26_1
-- make=4.3=h516909a_0
-- matplotlib=3.1.1=py37_0
-- nc-time-axis=1.2.0=py_1
-- netcdf-fortran=4.4.5=hea25ff8_1000
-- netcdf4=1.5.1.2=py37had58050_0
-- numba=0.48.0=py37hb3f55d8_0
-- pandas=1.0.4=py37h0da4684_0
-- phantomjs=2.1.1=1 # for geoviews
-- pillow=7.1.2=py37h718be6c_0
-- pip=20.1.1=py_1
-- plotly=4.8.1=pyh9f0ad1d_0
-- pyinterp=0.3.2=openblas_py37hcf60e08_2
-- pyshp=2.1.0=py_0
-- python=3.7.3=h5b0a415_0
-- pyyaml=5.3.1=py37h8f50634_0
-- rasterio=1.1.1=py37h900e953_0
-- regionmask=0.5.0=py_1
-- rtree=0.9.4=py37h8526d28_1
-- scikit-image=0.17.2=py37h0da4684_1
-- scikit-learn=0.23.1=py37h8a51577_0
-- scipy=1.4.1=py37ha3d9a3c_3
-- seaborn=0.10.1=1
-- selenium=3.141.0=py37h8f50634_1001 # for geoviews
-- shapely=1.7.0=py37hb106bac_1
-- sparse=0.9.1=py_0
-- statsmodels=0.11.1=py37h8f50634_2
-- tini=0.18.0=h14c3975_1001
-- xarray=0.15.1=py_0
-- xesmf=0.3.0=py_0
-- xhistogram=0.1.1=py_0
-- xlrd=1.2.0=py_0
-- zarr=2.4.0=py_0
-- zeromq=4.3.2=he1b5a44_2
-- pip:
-  - py-noaa
-  - sidecar
-  - climate-toolbox
-  - parameterize-jobs
-  - rhg_compute_tools
+  - beautifulsoup4
+  - blas
+  - bottleneck
+  - boto3
+  - bqplot
+  - bump2version
+  - cachecontrol
+  - cartopy
+  - cftime
+  - click
+  - compilers
+  - dask
+  - dask-glm
+  - dask-kubernetes
+  - dask-ml
+  - datashader
+  - distributed
+  - dropbox
+  - esmpy
+  - fastparquet
+  - fiona
+  - fusepy
+  - gcsfs
+  - gdal
+  - geopandas
+  - geotiff
+  - geoviews
+  - git
+  - gitpython
+  - google-cloud-container
+  - google-cloud-storage
+  - holoviews
+  - h5netcdf
+  - intake-esm
+  - jedi
+  - jupyter-server-proxy          # need server proxy on workers if using remote scheduler
+  - kubernetes
+  - lz4
+  - make
+  - matplotlib
+  - nc-time-axis
+  - netcdf-fortran
+  - netcdf4
+  - numba
+  - pandas
+  - phantomjs       # for geoviews
+  - pillow
+  - pip
+  - plotly
+  - pyinterp
+  - pyshp
+  - python
+  - pyyaml
+  - rasterio
+  - regionmask
+  - rtree
+  - scikit-image
+  - scikit-learn
+  - scipy
+  - seaborn
+  - selenium                         # for geoviews
+  - shapely
+  - sparse
+  - statsmodels
+  - tini
+  - xarray
+  - xesmf
+  - xhistogram
+  - xlrd
+  - zarr
+  - zeromq
+  - pip:
+    - py-noaa
+    - sidecar
+    - climate-toolbox
+    - parameterize-jobs
+    - rhg_compute_tools
 name: base
diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml
index 704df48..03fc43e 100644
--- a/octave-worker/octave_environment.yml
+++ b/octave-worker/octave_environment.yml
@@ -2,5 +2,5 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - octave_kernel=0.31.0=py_0
-  - oct2py=5.0.4=py_0
+  - octave_kernel
+  - oct2py
diff --git a/pin.py b/pin.py
index 591ddd2..175ab22 100644
--- a/pin.py
+++ b/pin.py
@@ -297,8 +297,8 @@ def pin(file, dry_run):
     spec_files = [
             ('base_environment.yml', 'base'),
             ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base')
-    ]
+            ('octave-worker/octave_environment.yml', 'base'),
+            ('notebook/r_environment.yml', 'r')]
     
     if file == 'all':
         pin_files(spec_files, dry_run=dry_run)
@@ -332,8 +332,8 @@ def unpin(file, dry_run):
     spec_files = [
             ('base_environment.yml', 'base'),
             ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base')
-    ]
+            ('octave-worker/octave_environment.yml', 'base'),
+            ('notebook/r_environment.yml', 'r')]
     
     if file == 'all':
         unpin_files(spec_files, dry_run=dry_run)

From e16aa2aa0900814cff18335587cd842095598d53 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 11:59:31 -0700
Subject: [PATCH 058/224] add pip back into notebook env to avoid warning

---
 notebook/notebook_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 7ddc919..8a8f870 100644
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -17,6 +17,7 @@ dependencies:
   - octave_kernel
   - oct2py
   - openssh
+  - pip
   - pytest
   - pytest-cov
   - pytest-runner

From b5c24bc00ae36d8d30b31ad192cdfb55ed2f4149 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 12:29:59 -0700
Subject: [PATCH 059/224] pin all

---
 base_environment.yml                 | 150 +++++++++++++--------------
 notebook/notebook_environment.yml    |  44 ++++----
 octave-worker/octave_environment.yml |   4 +-
 3 files changed, 99 insertions(+), 99 deletions(-)
 mode change 100644 => 100755 base_environment.yml
 mode change 100644 => 100755 notebook/notebook_environment.yml
 mode change 100644 => 100755 octave-worker/octave_environment.yml

diff --git a/base_environment.yml b/base_environment.yml
old mode 100644
new mode 100755
index 0aa73ea..1353353
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -1,81 +1,81 @@
 channels:
   - conda-forge
 dependencies:
-  - beautifulsoup4
-  - blas
-  - bottleneck
-  - boto3
-  - bqplot
-  - bump2version
-  - cachecontrol
-  - cartopy
-  - cftime
-  - click
-  - compilers
-  - dask
-  - dask-glm
-  - dask-kubernetes
-  - dask-ml
-  - datashader
-  - distributed
-  - dropbox
-  - esmpy
-  - fastparquet
-  - fiona
-  - fusepy
-  - gcsfs
-  - gdal
-  - geopandas
-  - geotiff
-  - geoviews
-  - git
-  - gitpython
-  - google-cloud-container
-  - google-cloud-storage
-  - holoviews
-  - h5netcdf
-  - intake-esm
-  - jedi
-  - jupyter-server-proxy          # need server proxy on workers if using remote scheduler
-  - kubernetes
-  - lz4
-  - make
-  - matplotlib
-  - nc-time-axis
-  - netcdf-fortran
-  - netcdf4
-  - numba
-  - pandas
-  - phantomjs       # for geoviews
-  - pillow
-  - pip
-  - plotly
-  - pyinterp
-  - pyshp
-  - python
-  - pyyaml
-  - rasterio
-  - regionmask
-  - rtree
-  - scikit-image
-  - scikit-learn
-  - scipy
-  - seaborn
-  - selenium                         # for geoviews
-  - shapely
-  - sparse
-  - statsmodels
-  - tini
-  - xarray
-  - xesmf
-  - xhistogram
-  - xlrd
-  - zarr
-  - zeromq
+  - beautifulsoup4=4.9.1=py37hc8dfbb8_0
+  - blas=2.14=openblas
+  - bottleneck=1.3.2=py37h03ebfcd_1
+  - boto3=1.14.3=pyh9f0ad1d_0
+  - bqplot=0.12.12=pyh9f0ad1d_0
+  - bump2version=1.0.0=pyh9f0ad1d_0
+  - cachecontrol=0.12.6=py_0
+  - cartopy=0.18.0=py37h4b180d9_0
+  - cftime=1.1.3=py37h03ebfcd_0
+  - click=7.1.2=pyh9f0ad1d_0
+  - compilers=1.0.4=0
+  - dask=2.18.1=py_0
+  - dask-glm=0.2.0=py_1
+  - dask-kubernetes=0.10.1=py_0
+  - dask-ml=1.5.0=py_0
+  - datashader=0.10.0=py_0
+  - distributed=2.18.0=py37hc8dfbb8_0
+  - dropbox=10.1.1=pyh9f0ad1d_0
+  - esmpy=8.0.0=nompi_py37hf0e99fa_1
+  - fastparquet=0.4.0=py37h03ebfcd_0
+  - fiona=1.8.13=py37h0492a4a_1
+  - fusepy=3.0.1=py_0
+  - gcsfs=0.6.2=py_0
+  - gdal=3.0.4=py37h4b180d9_6
+  - geopandas=0.7.0=py_1
+  - geotiff=1.5.1=h05acad5_10
+  - geoviews=1.8.1=py_0
+  - git=2.27.0=pl526h5e3e691_0
+  - gitpython=3.1.3=py_0
+  - google-cloud-container=0.3.0=py37_0
+  - google-cloud-storage=1.28.1=pyh9f0ad1d_0
+  - holoviews=1.13.2=pyh9f0ad1d_0
+  - h5netcdf=0.8.0=py_0
+  - intake-esm=2020.6.11=py_0
+  - jedi=0.17.0=py37hc8dfbb8_0
+  - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
+  - kubernetes=1.18.3=0
+  - lz4=3.1.0=py37h5a7ed16_0
+  - make=4.3=h516909a_0
+  - matplotlib=3.2.1=0
+  - nc-time-axis=1.2.0=py_1
+  - netcdf-fortran=4.5.2=nompi_h45d7149_104
+  - netcdf4=1.5.3=nompi_py37hec16513_103
+  - numba=0.49.1=py37h0da4684_0
+  - pandas=1.0.4=py37h0da4684_0
+  - phantomjs=2.1.1=1 # for geoviews
+  - pillow=7.1.2=py37h718be6c_0
+  - pip=20.1.1=py_1
+  - plotly=4.8.1=pyh9f0ad1d_0
+  - pyinterp=0.3.2=mkl_py37he448baa_102
+  - pyshp=2.1.0=py_0
+  - python=3.7.6=cpython_h8356626_6
+  - pyyaml=5.3.1=py37h8f50634_0
+  - rasterio=1.1.5=py37h0492a4a_0
+  - regionmask=0.5.0=py_1
+  - rtree=0.9.4=py37h8526d28_1
+  - scikit-image=0.17.2=py37h0da4684_1
+  - scikit-learn=0.23.1=py37h8a51577_0
+  - scipy=1.4.1=py37ha3d9a3c_3
+  - seaborn=0.10.1=1
+  - selenium=3.141.0=py37h8f50634_1001 # for geoviews
+  - shapely=1.7.0=py37hc88ce51_3
+  - sparse=0.10.0=py_0
+  - statsmodels=0.11.1=py37h8f50634_2
+  - tini=0.18.0=h14c3975_1001
+  - xarray=0.15.1=py_0
+  - xesmf=0.3.0=py_0
+  - xhistogram=0.1.1=py_0
+  - xlrd=1.2.0=py_0
+  - zarr=2.4.0=py_0
+  - zeromq=4.3.2=he1b5a44_2
   - pip:
-    - py-noaa
-    - sidecar
-    - climate-toolbox
-    - parameterize-jobs
+    - py-noaa==1.0
+    - sidecar==0.4.0
+    - climate-toolbox==0.1.5
+    - parameterize-jobs==0.1.1
     - rhg_compute_tools
 name: base
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
old mode 100644
new mode 100755
index 8a8f870..0b945be
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -2,27 +2,27 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - black
-  - coverage
-  - dask-labextension
-  - flake8
-  - ipdb
-  - ipyleaflet
-  - ipympl
-  - isort
-  - jupyterlab_code_formatter
-  - nano
-  - nb_conda_kernels
-  - nose
-  - octave_kernel
-  - oct2py
-  - openssh
-  - pip
-  - pytest
-  - pytest-cov
-  - pytest-runner
-  - python-graphviz
-  - sphinx
-  - tox
+  - black=19.10b0=py37_0
+  - coverage=5.1=py37h8f50634_0
+  - dask-labextension=2.0.2=py_0
+  - flake8=3.8.3=pyh9f0ad1d_0
+  - ipdb=0.13.0=pyh9f0ad1d_0
+  - ipyleaflet=0.13.0=pyh9f0ad1d_0
+  - ipympl=0.5.6=pyh9f0ad1d_1
+  - isort=4.3.21=py37hc8dfbb8_1
+  - jupyterlab_code_formatter=1.3.1=py_0
+  - nano=2.9.8=h969982e_1000
+  - nb_conda_kernels=2.2.3=py37_0
+  - nose=1.3.7=py37hc8dfbb8_1004
+  - octave_kernel=0.32.0=pyh9f0ad1d_0
+  - oct2py=5.0.4=py_0
+  - openssh=8.3p1=h5957347_0
+  - pip=20.1.1=py_1
+  - pytest=5.4.3=py37hc8dfbb8_0
+  - pytest-cov=2.10.0=pyh9f0ad1d_0
+  - pytest-runner=5.2=py_0
+  - python-graphviz=0.14=pyh9f0ad1d_0
+  - sphinx=3.1.1=py_0
+  - tox=3.15.2=py37hc8dfbb8_0
   - pip:
     - black_nbconvert
diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml
old mode 100644
new mode 100755
index 03fc43e..7585324
--- a/octave-worker/octave_environment.yml
+++ b/octave-worker/octave_environment.yml
@@ -2,5 +2,5 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - octave_kernel
-  - oct2py
+  - octave_kernel=0.32.0=pyh9f0ad1d_0
+  - oct2py=5.0.4=py_0

From 63811b41d8279ec50c291e6e42dcd2a96029c90a Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 13:26:49 -0700
Subject: [PATCH 060/224] update travis yml

---
 .travis.yml | 28 ++--------------------------
 1 file changed, 2 insertions(+), 26 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7032e46..30c9fb0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,3 @@
-sudo: required
-
 services:
   - docker
 
@@ -31,7 +29,6 @@ deploy:
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" &&
     docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH"
-  skip_cleanup: true
   on:
     all_branches: true
     condition: $TRAVIS_BRANCH =~ ^dev
@@ -44,7 +41,6 @@ deploy:
     docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" &&
     docker push "rhodium/$IMAGE_NAME:dev" &&
     docker push "rhodium/$IMAGE_NAME:latest"
-  skip_cleanup: true
   on:
     branch: master
 
@@ -54,7 +50,6 @@ deploy:
     rhodium/$IMAGE_NAME:$TRAVIS_TAG &&
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG"
-  skip_cleanup: true
   on:
     tags: true
 
@@ -68,11 +63,9 @@ deploy:
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/octave-worker:$TRAVIS_COMMIT" &&
     docker push "rhodium/octave-worker:$TRAVIS_BRANCH"
-  skip_cleanup: true
   on:
     all_branches: true
-    condition: $TRAVIS_BRANCH =~ ^dev
-    condition: $IMAGE_NAME = worker
+    condition: ($TRAVIS_BRANCH =~ ^dev) AND ($IMAGE_NAME = worker)
 
 - provider: script
   script: >-
@@ -84,7 +77,6 @@ deploy:
     docker push "rhodium/octave-worker:$TRAVIS_COMMIT" &&
     docker push "rhodium/octave-worker:dev" &&
     docker push "rhodium/octave-worker:latest"
-  skip_cleanup: true
   on:
     branch: master
     condition: $IMAGE_NAME = worker
@@ -95,22 +87,6 @@ deploy:
     rhodium/octave-worker:$TRAVIS_TAG &&
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/octave-worker:$TRAVIS_TAG"
-  skip_cleanup: true
   on:
     tags: true
-    condition: $IMAGE_NAME = worker
-
-  # - stage: alignment
-  #   language: python
-  #   python:
-  #     - 3.6
-  #   script:
-  #     - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-  #     - bash miniconda.sh -b -p $HOME/miniconda
-  #     - export PATH="$HOME/miniconda/bin:$PATH"
-  #     - hash -r
-  #     - conda config --set always_yes yes --set changeps1 no
-  #     - conda update -q conda
-  #     - conda info -a
-  #     - conda install pytest pytest-cov pyyaml
-  #     - pytest
+    condition: $IMAGE_NAME = worker
\ No newline at end of file

From 919317b253c2b24119315233a57b96deb2e7b966 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 14:07:45 -0700
Subject: [PATCH 061/224] fix travis condition syntax

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 30c9fb0..6816a1c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -65,7 +65,7 @@ deploy:
     docker push "rhodium/octave-worker:$TRAVIS_BRANCH"
   on:
     all_branches: true
-    condition: ($TRAVIS_BRANCH =~ ^dev) AND ($IMAGE_NAME = worker)
+    condition: ("$TRAVIS_BRANCH" =~ "^dev") && ("$IMAGE_NAME" == "worker")
 
 - provider: script
   script: >-

From 32ec9f392f3a063317c072ccfe8f4b606f2491cb Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 16 Jun 2020 14:35:27 -0700
Subject: [PATCH 062/224] remove labextension files already installed

---
 notebook/Dockerfile | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 3553ed0..b986b8f 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -66,12 +66,9 @@ RUN jupyter labextension update --all && \
     jupyter labextension install \
     @bokeh/jupyter_bokeh \
     @jupyter-widgets/jupyterlab-manager \
-    @jupyter-widgets/jupyterlab-sidecar \
     @pyviz/jupyterlab_pyviz \
     @ryantam626/jupyterlab_code_formatter \
     dask-labextension \
-    jupyter-leaflet \
-    jupyter-matplotlib \
     jupyterlab-plotly \
     plotlywidget && \
     jupyter serverextension enable --py jupyterlab_code_formatter

From 7aa52b39a308418707e290e55716cca695fb86c5 Mon Sep 17 00:00:00 2001
From: dpa9694 <danielpaulallen@gmail.com>
Date: Tue, 16 Jun 2020 19:52:58 -0700
Subject: [PATCH 063/224] Download vdatum to appropriate path (#127)

* specify vdatum path

* update vdatum install loc

Co-authored-by: Ian Bolliger <bolliger@berkeley.edu>
---
 common.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/common.sh b/common.sh
index 3d01f89..6b97bcc 100644
--- a/common.sh
+++ b/common.sh
@@ -38,8 +38,8 @@ chmod +x /usr/bin/cloud_sql_proxy
 # download vdatum
 wget https://vdatum.noaa.gov/download/data/vdatum_v4.0.1.zip -O /usr/bin/vdatum.zip
 wget https://vdatum.noaa.gov/download/data/vdatum_EGM1996.zip -O /usr/bin/vdatum_EGM1996.zip
-unzip /usr/bin/vdatum.zip
-unzip -o /usr/bin/vdatum_EGM1996.zip
+unzip /usr/bin/vdatum.zip -d /usr/bin
+unzip -o /usr/bin/vdatum_EGM1996.zip -d /usr/bin
 rm /usr/bin/vdatum.zip
 rm /usr/bin/vdatum_EGM1996.zip
 
@@ -47,4 +47,3 @@ rm /usr/bin/vdatum_EGM1996.zip
 chmod +x /usr/bin/prepare.sh
 mkdir /gcs
 mkdir /opt/app
-

From 6d9fccb86c9f4fdffb32baf056eb6d454079bd3e Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 3 Jul 2020 06:51:39 -0700
Subject: [PATCH 064/224] add notebook linenums and ruler

---
 notebook/overrides.json | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/notebook/overrides.json b/notebook/overrides.json
index 78dcf65..38e8912 100644
--- a/notebook/overrides.json
+++ b/notebook/overrides.json
@@ -1,8 +1,14 @@
 {
   "@jupyterlab/fileeditor-extension:plugin": {
     "editorConfig": {
-        "rulers": [79],
+        "rulers": [88],
         "lineNumbers": true
     }
+  },
+  "@jupyterlab/notebook-extension:tracker": {
+    "codeCellConfig": {
+      "rulers": [88],
+      "lineNumbers": true
+    }
   }
 }

From c5eb26c0f5de45e59933220dceb36a4782b895ea Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 3 Jul 2020 06:52:03 -0700
Subject: [PATCH 065/224] ignore vscode settings

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 79e93af..765dc93 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ worker-config.yml
 .coverage
 __pycache__
 .pytest_cache
+.vscode/settings.json

From f89bcc482c3e6339e38757af32b9d8ee2088c6a1 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 3 Jul 2020 09:35:50 -0700
Subject: [PATCH 066/224] make travis.yml more readable

---
 .travis.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 6816a1c..c927d37 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,11 +6,14 @@ env:
 - IMAGE_NAME=worker
 
 install:
-- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak \
-'s/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' \
-notebook/worker-template.yml; else sed -i.bak \
-'s/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' \
-notebook/worker-template.yml; fi"
+- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
+    sed -i.bak \
+      's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' \
+      notebook/worker-template.yml; 
+  else
+    sed -i.bak \
+      's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' \
+      notebook/worker-template.yml; fi"
 - "rm notebook/worker-template.yml.bak"
 - "cat notebook/worker-template.yml | grep image:"
 - "cp base_environment.yml $IMAGE_NAME/base_environment.yml"

From b74d5b8f0ce4287191ccb15748059b9c3078e165 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 3 Jul 2020 09:36:01 -0700
Subject: [PATCH 067/224] update pins with xarray=0.14

---
 base_environment.yml              | 45 +++++++++++++++----------------
 notebook/notebook_environment.yml |  7 +++--
 2 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/base_environment.yml b/base_environment.yml
index 1353353..9c0a4eb 100755
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -4,52 +4,50 @@ dependencies:
   - beautifulsoup4=4.9.1=py37hc8dfbb8_0
   - blas=2.14=openblas
   - bottleneck=1.3.2=py37h03ebfcd_1
-  - boto3=1.14.3=pyh9f0ad1d_0
-  - bqplot=0.12.12=pyh9f0ad1d_0
-  - bump2version=1.0.0=pyh9f0ad1d_0
+  - boto3=1.14.16=pyh9f0ad1d_0
+  - bqplot=0.12.13=pyh9f0ad1d_0
   - cachecontrol=0.12.6=py_0
   - cartopy=0.18.0=py37h4b180d9_0
   - cftime=1.1.3=py37h03ebfcd_0
   - click=7.1.2=pyh9f0ad1d_0
   - compilers=1.0.4=0
-  - dask=2.18.1=py_0
+  - dask=2.20.0=py_0
   - dask-glm=0.2.0=py_1
   - dask-kubernetes=0.10.1=py_0
   - dask-ml=1.5.0=py_0
   - datashader=0.10.0=py_0
-  - distributed=2.18.0=py37hc8dfbb8_0
+  - distributed=2.20.0=py37hc8dfbb8_0
   - dropbox=10.1.1=pyh9f0ad1d_0
-  - esmpy=8.0.0=nompi_py37hf0e99fa_1
+  - esmpy=8.0.1=nompi_py37h777d1d2_0
   - fastparquet=0.4.0=py37h03ebfcd_0
   - fiona=1.8.13=py37h0492a4a_1
   - fusepy=3.0.1=py_0
   - gcsfs=0.6.2=py_0
-  - gdal=3.0.4=py37h4b180d9_6
-  - geopandas=0.7.0=py_1
-  - geotiff=1.5.1=h05acad5_10
+  - gdal=3.0.4=py37h4b180d9_10
+  - geopandas=0.8.0=py_1
+  - geotiff=1.6.0=h05acad5_0
   - geoviews=1.8.1=py_0
   - git=2.27.0=pl526h5e3e691_0
-  - gitpython=3.1.3=py_0
   - google-cloud-container=0.3.0=py37_0
   - google-cloud-storage=1.28.1=pyh9f0ad1d_0
-  - holoviews=1.13.2=pyh9f0ad1d_0
+  - holoviews=1.13.3=pyh9f0ad1d_0
   - h5netcdf=0.8.0=py_0
-  - intake-esm=2020.6.11=py_0
-  - jedi=0.17.0=py37hc8dfbb8_0
+  - intake-esm=2020.3.16.2=py_0
+  - jedi=0.17.1=py37hc8dfbb8_0
   - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
-  - kubernetes=1.18.3=0
+  - kubernetes=1.18.5=0
   - lz4=3.1.0=py37h5a7ed16_0
   - make=4.3=h516909a_0
-  - matplotlib=3.2.1=0
+  - matplotlib=3.2.2=0
   - nc-time-axis=1.2.0=py_1
-  - netcdf-fortran=4.5.2=nompi_h45d7149_104
-  - netcdf4=1.5.3=nompi_py37hec16513_103
+  - netcdf-fortran=4.5.3=nompi_hfef6a68_100
+  - netcdf4=1.5.3=nompi_py37hdc49583_105
   - numba=0.49.1=py37h0da4684_0
-  - pandas=1.0.4=py37h0da4684_0
+  - pandas=1.0.5=py37h0da4684_0
   - phantomjs=2.1.1=1 # for geoviews
-  - pillow=7.1.2=py37h718be6c_0
+  - pillow=7.2.0=py37h718be6c_0
   - pip=20.1.1=py_1
-  - plotly=4.8.1=pyh9f0ad1d_0
+  - plotly=4.8.2=pyh9f0ad1d_0
   - pyinterp=0.3.2=mkl_py37he448baa_102
   - pyshp=2.1.0=py_0
   - python=3.7.6=cpython_h8356626_6
@@ -59,23 +57,22 @@ dependencies:
   - rtree=0.9.4=py37h8526d28_1
   - scikit-image=0.17.2=py37h0da4684_1
   - scikit-learn=0.23.1=py37h8a51577_0
-  - scipy=1.4.1=py37ha3d9a3c_3
+  - scipy=1.5.0=py37ha3d9a3c_0
   - seaborn=0.10.1=1
   - selenium=3.141.0=py37h8f50634_1001 # for geoviews
   - shapely=1.7.0=py37hc88ce51_3
   - sparse=0.10.0=py_0
   - statsmodels=0.11.1=py37h8f50634_2
   - tini=0.18.0=h14c3975_1001
-  - xarray=0.15.1=py_0
+  - xarray=0.14.1=py_1                 # pinkeep: xarray=0.14
   - xesmf=0.3.0=py_0
   - xhistogram=0.1.1=py_0
   - xlrd=1.2.0=py_0
   - zarr=2.4.0=py_0
   - zeromq=4.3.2=he1b5a44_2
   - pip:
-    - py-noaa==1.0
-    - sidecar==0.4.0
     - climate-toolbox==0.1.5
+    - noaa-coops==0.1.5
     - parameterize-jobs==0.1.1
     - rhg_compute_tools
 name: base
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 0b945be..e9a0191 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -3,11 +3,13 @@ channels:
   - conda-forge
 dependencies:
   - black=19.10b0=py37_0
+  - bump2version=1.0.0=pyh9f0ad1d_0
   - coverage=5.1=py37h8f50634_0
   - dask-labextension=2.0.2=py_0
   - flake8=3.8.3=pyh9f0ad1d_0
+  - gitpython=3.1.3=py_0
   - ipdb=0.13.0=pyh9f0ad1d_0
-  - ipyleaflet=0.13.0=pyh9f0ad1d_0
+  - ipyleaflet=0.13.1=pyh9f0ad1d_0
   - ipympl=0.5.6=pyh9f0ad1d_1
   - isort=4.3.21=py37hc8dfbb8_1
   - jupyterlab_code_formatter=1.3.1=py_0
@@ -22,7 +24,8 @@ dependencies:
   - pytest-cov=2.10.0=pyh9f0ad1d_0
   - pytest-runner=5.2=py_0
   - python-graphviz=0.14=pyh9f0ad1d_0
+  - sidecar=0.4.0=pyh9f0ad1d_0
   - sphinx=3.1.1=py_0
-  - tox=3.15.2=py37hc8dfbb8_0
+  - tox=3.16.1=py37hc8dfbb8_0
   - pip:
     - black_nbconvert

From f20a51cb2f7540e175be95538940fe983de61d6d Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 3 Jul 2020 10:07:33 -0700
Subject: [PATCH 068/224] pass debian_frontend to sudo in dockerfiles

---
 notebook/Dockerfile      | 2 +-
 octave-worker/Dockerfile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index b986b8f..f99307c 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -36,7 +36,7 @@ RUN sudo chown -R $NB_USER /gcs
 
 
 ## more apt-get
-RUN sudo apt-get install -yq \
+RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
   vim less man locate kmod dialog \
   octave && \
   sudo apt-get clean
diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile
index 75039f0..4ac3655 100644
--- a/octave-worker/Dockerfile
+++ b/octave-worker/Dockerfile
@@ -6,7 +6,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 COPY octave_environment.yml /opt/conda/specs/octave_environment.yml
 
 ## install octave
-RUN sudo apt-get update && sudo apt-get install -yq octave
+RUN sudo DEBIAN_FRONTEND=noninteractive apt-get update && sudo apt-get install -yq octave
 
 # add octave-specific packages
 RUN conda env update -f /opt/conda/specs/octave_environment.yml

From 0e1fffc248c3eb14b6b7e69d4a20722dde6c3ca6 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 3 Jul 2020 13:04:43 -0700
Subject: [PATCH 069/224] enable code formatter system wide so no overwrite

---
 notebook/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index f99307c..cd9d245 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -71,7 +71,7 @@ RUN jupyter labextension update --all && \
     dask-labextension \
     jupyterlab-plotly \
     plotlywidget && \
-    jupyter serverextension enable --py jupyterlab_code_formatter
+    jupyter serverextension enable --py jupyterlab_code_formatter --system
 
 
 ## clean up

From 1996de1cb7ffbaeaf8320de713853202de33294b Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 3 Jul 2020 14:27:09 -0700
Subject: [PATCH 070/224] sudo the serverextension in notebook

---
 notebook/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index cd9d245..9c4829c 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -71,7 +71,7 @@ RUN jupyter labextension update --all && \
     dask-labextension \
     jupyterlab-plotly \
     plotlywidget && \
-    jupyter serverextension enable --py jupyterlab_code_formatter --system
+    sudo jupyter serverextension enable --py jupyterlab_code_formatter --system
 
 
 ## clean up

From 85597c826e40130ed2aa12caf69aa435f1a106bf Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 5 Jul 2020 16:50:59 -0700
Subject: [PATCH 071/224] try triggering clawpack build

---
 .travis.yml       | 13 ++++++++++---
 build_clawpack.js | 49 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 3 deletions(-)
 create mode 100644 build_clawpack.js

diff --git a/.travis.yml b/.travis.yml
index c927d37..b4f558c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,8 +2,12 @@ services:
   - docker
 
 env:
-- IMAGE_NAME=notebook
-- IMAGE_NAME=worker
+  matrix:
+    - IMAGE_NAME=notebook
+    - IMAGE_NAME=worker  
+  global:
+    secure: FD1HCYWd/H7VDC2O+bWkJP4WooHigMW0XWqcbKZ+M5dYwVzzDeAgds36CT6Le+kREvKbmaDPxGDCoW33Ceu96lgZWla1Le0YjdJyiyvx34dGJnX3IrucRXbvc3+UN+qwfOj8UMadPGexVwcgcWSTIc0fwdibW9fE020fQF47XgxsXU5Ouro0e0ktvqfTs2revr2sU5aMDyxs4wyfVo8K905LfrqGSbIBSPCcjxLK9clByRNwwr4rWfSocBKnIF9xq6H4piDnC67rWfymTW4Lr9H9AMbH/kVErTAc+RACgmjVKMKYhP2DHk2N3SwE7bzcCgEkgj6LoC/8mogrY0Sml6Wa2zvNuFBzsSgocb07fGV/cPCFzgmqI6fLNtmDdrK5t5MEAvFB3XhsNgI8S04Rtzgijfp5tEEs53JAOjty3Uajv+pnrJOSP+ilo4VxB0aTpFJG0k+aCHLFY0lUuxWKEkgo3xRAhOgXmg1qbetarwR7EF4jQCEwARFlipbBHh8aJdTYSpqmLbrkzVU4luP3ycdFCea4b4HNfhC/OdnV+0j6NE8LEV29Sv8+QNbDeYJuexLJ3Kjo3BWkR1rVCa+ciqDoYBriPPTmQ1S/tj2ttFVZubyTx4u2yDP0ZMxeJsJvmCbFh2N+LujO4xTxadTcLMxadvCOJkB9w1i5XiO1Zg4=
+
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
@@ -92,4 +96,7 @@ deploy:
     docker push "rhodium/octave-worker:$TRAVIS_TAG"
   on:
     tags: true
-    condition: $IMAGE_NAME = worker
\ No newline at end of file
+    condition: $IMAGE_NAME = worker
+
+after_deploy:
+  - node build_clawpack.js
\ No newline at end of file
diff --git a/build_clawpack.js b/build_clawpack.js
new file mode 100644
index 0000000..ff9c466
--- /dev/null
+++ b/build_clawpack.js
@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+"use strict";
+
+const shell = require('shelljs'),
+  path = require('path'),
+  got = require('got');
+
+
+console.log(`Fetching Git commit hash...`);
+
+const gitCommitRet = shell.exec('git rev-parse HEAD', {
+  cwd: path.join(__dirname, '..')
+});
+
+if (0 !== gitCommitRet.code) {
+  console.error('Error getting git commit hash');
+  
+  process.exit(-1);
+}
+
+const gitCommitHash = gitCommitRet.stdout.trim();
+
+console.log(`Git commit: ${gitCommitHash}`);
+
+console.log('Calling Travis...');
+
+
+got.post(`https://api.travis-ci.org/repo/climateimpactlab%2Fclawpack/requests`, {
+  headers: {
+    "Content-Type": "application/json",
+    "Accept": "application/json",
+    "Travis-API-Version": "3",
+    "Authorization": `token ${process.env.TRAVIS_API_TOKEN}`,
+  },
+  body: JSON.stringify({
+    request: {
+      message: `Trigger build at ClimateImpactLab/clawpack commit: ${gitCommitHash}`,
+      branch: 'source',
+    },    
+  }),
+})
+.then(() => {
+  console.log("Triggered build of ClimateImpactLab/clawpack");
+})
+.catch((err) => {
+  console.error(err);
+  
+  process.exit(-1);
+});
\ No newline at end of file

From 6084e2c4d84a48780ca369fd3950301ad41d99db Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 5 Jul 2020 16:53:38 -0700
Subject: [PATCH 072/224] only build clawpack from notebook

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index b4f558c..9a41b45 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -99,4 +99,4 @@ deploy:
     condition: $IMAGE_NAME = worker
 
 after_deploy:
-  - node build_clawpack.js
\ No newline at end of file
+  - if [ "${IMAGE_NAME}" = "notebook" ]; then node build_clawpack.js; fi
\ No newline at end of file

From f1aa2af6c375f040fc8c99080cc419dda35727f8 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 5 Jul 2020 16:55:17 -0700
Subject: [PATCH 073/224] should build after worker deployed not notbook

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 9a41b45..dc00506 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -99,4 +99,4 @@ deploy:
     condition: $IMAGE_NAME = worker
 
 after_deploy:
-  - if [ "${IMAGE_NAME}" = "notebook" ]; then node build_clawpack.js; fi
\ No newline at end of file
+  - if [ "${IMAGE_NAME}" = "worker" ]; then node build_clawpack.js; fi
\ No newline at end of file

From f940b2bb5437a94ac14bb0c244d14d6718412ab9 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 5 Jul 2020 20:06:02 -0700
Subject: [PATCH 074/224] include clawpack in build

---
 .gitmodules       |  3 +++
 worker/Dockerfile | 27 +++++++++++++++++++++++++++
 worker/clawpack   |  1 +
 3 files changed, 31 insertions(+)
 create mode 100644 .gitmodules
 create mode 160000 worker/clawpack

diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..d8d5431
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "worker/clawpack"]
+	path = worker/clawpack
+	url = git@github.com:ClimateImpactLab/clawpack.git
diff --git a/worker/Dockerfile b/worker/Dockerfile
index fe6d7df..0a4d100 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -53,4 +53,31 @@ ENV OMP_NUM_THREADS=1
 ENV MKL_NUM_THREADS=1
 ENV OPENBLAS_NUM_THREADS=1
 
+##################
+## clawpack added for coastal-specific image
+##################
+
+COPY . /clawpack
+
+## install Clawpack
+ENV CLAW=/clawpack
+ENV NETCDF4_DIR=/opt/conda
+ENV FC=gfortran
+ENV MPLBACKEND=Agg
+
+# this is needed to find libraries when building geoclaw (particularly lapack)
+ENV LIB_PATHS=/opt/conda/lib
+
+WORKDIR /clawpack
+
+# need to change shell in order for source command to work
+SHELL ["/bin/bash", "-c"]
+
+# install clawpack
+RUN pip install -e .
+
+WORKDIR /
+
+##################
+
 ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
diff --git a/worker/clawpack b/worker/clawpack
new file mode 160000
index 0000000..c854612
--- /dev/null
+++ b/worker/clawpack
@@ -0,0 +1 @@
+Subproject commit c85461231106286f9ba037261bccfe1fb2f76bac

From ceca9545d790f25aa2700236f86bf315109194d3 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 5 Jul 2020 20:08:04 -0700
Subject: [PATCH 075/224] forgot to include new packages

---
 .travis.yml          | 13 +++----------
 base_environment.yml |  2 ++
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index dc00506..c927d37 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,12 +2,8 @@ services:
   - docker
 
 env:
-  matrix:
-    - IMAGE_NAME=notebook
-    - IMAGE_NAME=worker  
-  global:
-    secure: FD1HCYWd/H7VDC2O+bWkJP4WooHigMW0XWqcbKZ+M5dYwVzzDeAgds36CT6Le+kREvKbmaDPxGDCoW33Ceu96lgZWla1Le0YjdJyiyvx34dGJnX3IrucRXbvc3+UN+qwfOj8UMadPGexVwcgcWSTIc0fwdibW9fE020fQF47XgxsXU5Ouro0e0ktvqfTs2revr2sU5aMDyxs4wyfVo8K905LfrqGSbIBSPCcjxLK9clByRNwwr4rWfSocBKnIF9xq6H4piDnC67rWfymTW4Lr9H9AMbH/kVErTAc+RACgmjVKMKYhP2DHk2N3SwE7bzcCgEkgj6LoC/8mogrY0Sml6Wa2zvNuFBzsSgocb07fGV/cPCFzgmqI6fLNtmDdrK5t5MEAvFB3XhsNgI8S04Rtzgijfp5tEEs53JAOjty3Uajv+pnrJOSP+ilo4VxB0aTpFJG0k+aCHLFY0lUuxWKEkgo3xRAhOgXmg1qbetarwR7EF4jQCEwARFlipbBHh8aJdTYSpqmLbrkzVU4luP3ycdFCea4b4HNfhC/OdnV+0j6NE8LEV29Sv8+QNbDeYJuexLJ3Kjo3BWkR1rVCa+ciqDoYBriPPTmQ1S/tj2ttFVZubyTx4u2yDP0ZMxeJsJvmCbFh2N+LujO4xTxadTcLMxadvCOJkB9w1i5XiO1Zg4=
-
+- IMAGE_NAME=notebook
+- IMAGE_NAME=worker
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
@@ -96,7 +92,4 @@ deploy:
     docker push "rhodium/octave-worker:$TRAVIS_TAG"
   on:
     tags: true
-    condition: $IMAGE_NAME = worker
-
-after_deploy:
-  - if [ "${IMAGE_NAME}" = "worker" ]; then node build_clawpack.js; fi
\ No newline at end of file
+    condition: $IMAGE_NAME = worker
\ No newline at end of file
diff --git a/base_environment.yml b/base_environment.yml
index 9c0a4eb..1ac7889 100755
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -71,8 +71,10 @@ dependencies:
   - zarr=2.4.0=py_0
   - zeromq=4.3.2=he1b5a44_2
   - pip:
+    - --no-cache-dir
     - climate-toolbox==0.1.5
     - noaa-coops==0.1.5
     - parameterize-jobs==0.1.1
     - rhg_compute_tools
+    - git+https://github.com/maritimeplanning/pytides.git@master
 name: base

From 78990f35fef21d3cfc12202ad02f809e44df179d Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 5 Jul 2020 20:23:33 -0700
Subject: [PATCH 076/224] change to https so travis works w/ clawpack

---
 .gitmodules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitmodules b/.gitmodules
index d8d5431..5d4ac05 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "worker/clawpack"]
 	path = worker/clawpack
-	url = git@github.com:ClimateImpactLab/clawpack.git
+	url = https://github.com/ClimateImpactLab/clawpack.git

From 82c6c7450e7995a2673a446b8698fccd25094c7d Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 5 Jul 2020 21:29:39 -0700
Subject: [PATCH 077/224] can't do --no-cache-dir

---
 base_environment.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/base_environment.yml b/base_environment.yml
index 1ac7889..9c97138 100755
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -71,7 +71,6 @@ dependencies:
   - zarr=2.4.0=py_0
   - zeromq=4.3.2=he1b5a44_2
   - pip:
-    - --no-cache-dir
     - climate-toolbox==0.1.5
     - noaa-coops==0.1.5
     - parameterize-jobs==0.1.1

From 7420d69b8d11ecfe99ea3fc9b12bdcbe0838f2c3 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 5 Jul 2020 21:53:35 -0700
Subject: [PATCH 078/224] copy correct clawpack folder in dockerfile

---
 worker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/worker/Dockerfile b/worker/Dockerfile
index 0a4d100..21a4421 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -57,7 +57,7 @@ ENV OPENBLAS_NUM_THREADS=1
 ## clawpack added for coastal-specific image
 ##################
 
-COPY . /clawpack
+COPY ./clawpack /clawpack
 
 ## install Clawpack
 ENV CLAW=/clawpack

From 1117e8f5a78a8df08cd9a991d038b821e3030ae7 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 09:21:28 -0700
Subject: [PATCH 079/224] update submods

---
 .gitmodules         |  4 ++--
 .travis.yml         |  1 +
 notebook/Dockerfile | 25 +++++++++++++++++++++++++
 worker/Dockerfile   | 10 ++++++----
 4 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index 5d4ac05..2377de3 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,3 @@
-[submodule "worker/clawpack"]
-	path = worker/clawpack
+[submodule "clawpack"]
+	path = clawpack
 	url = https://github.com/ClimateImpactLab/clawpack.git
diff --git a/.travis.yml b/.travis.yml
index c927d37..7d40c60 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,6 +18,7 @@ install:
 - "cat notebook/worker-template.yml | grep image:"
 - "cp base_environment.yml $IMAGE_NAME/base_environment.yml"
 - "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh"
+- "cp -r clawpack $IMAGE_NAME/clawpack"
 - "cd $IMAGE_NAME"
 
 
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 9c4829c..ee642fc 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -73,6 +73,31 @@ RUN jupyter labextension update --all && \
     plotlywidget && \
     sudo jupyter serverextension enable --py jupyterlab_code_formatter --system
 
+##################
+## clawpack added for coastal-specific image
+##################
+
+COPY ./clawpack /clawpack
+
+## install Clawpack
+ENV CLAW=/clawpack
+ENV NETCDF4_DIR=/opt/conda
+ENV FC=gfortran
+
+WORKDIR /clawpack
+
+# need to change shell in order for source command to work
+SHELL ["/bin/bash", "-c"]
+
+# super sketchy hack to get around our need for compiler_compat binaries and some
+# other things that cause problems together?
+# see https://github.com/ContinuumIO/anaconda-issues/issues/11152
+RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
+
+# install clawpack
+RUN pip install -e .
+
+##################
 
 ## clean up
 RUN sudo rm -rf /var/lib/apt/lists/* /tempdir
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 21a4421..e7e305a 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -2,6 +2,7 @@
 FROM ubuntu:bionic-20200403@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
 ARG DEBIAN_FRONTEND=noninteractive
 ENV NPY_DISTUTILS_APPEND_FLAGS=1
+ENV MPLBACKEND=Agg
 
 ## filepath curation
 RUN mkdir /tempdir
@@ -63,16 +64,17 @@ COPY ./clawpack /clawpack
 ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
-ENV MPLBACKEND=Agg
-
-# this is needed to find libraries when building geoclaw (particularly lapack)
-ENV LIB_PATHS=/opt/conda/lib
 
 WORKDIR /clawpack
 
 # need to change shell in order for source command to work
 SHELL ["/bin/bash", "-c"]
 
+# super sketchy hack to get around our need for compiler_compat binaries and some
+# other things that cause problems together?
+# see https://github.com/ContinuumIO/anaconda-issues/issues/11152
+RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
+
 # install clawpack
 RUN pip install -e .
 

From 3121d8db39d7203c96637920595b86a902910083 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 09:21:50 -0700
Subject: [PATCH 080/224] no worker/clawpack

---
 worker/clawpack | 1 -
 1 file changed, 1 deletion(-)
 delete mode 160000 worker/clawpack

diff --git a/worker/clawpack b/worker/clawpack
deleted file mode 160000
index c854612..0000000
--- a/worker/clawpack
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit c85461231106286f9ba037261bccfe1fb2f76bac

From 8c2b66879f78f77e57668824c38cd20e8e0faf34 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 09:27:35 -0700
Subject: [PATCH 081/224] rm gitmodules

---
 .gitmodules       |  3 ---
 build_clawpack.js | 49 -----------------------------------------------
 2 files changed, 52 deletions(-)
 delete mode 100644 .gitmodules
 delete mode 100644 build_clawpack.js

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 2377de3..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "clawpack"]
-	path = clawpack
-	url = https://github.com/ClimateImpactLab/clawpack.git
diff --git a/build_clawpack.js b/build_clawpack.js
deleted file mode 100644
index ff9c466..0000000
--- a/build_clawpack.js
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env node
-"use strict";
-
-const shell = require('shelljs'),
-  path = require('path'),
-  got = require('got');
-
-
-console.log(`Fetching Git commit hash...`);
-
-const gitCommitRet = shell.exec('git rev-parse HEAD', {
-  cwd: path.join(__dirname, '..')
-});
-
-if (0 !== gitCommitRet.code) {
-  console.error('Error getting git commit hash');
-  
-  process.exit(-1);
-}
-
-const gitCommitHash = gitCommitRet.stdout.trim();
-
-console.log(`Git commit: ${gitCommitHash}`);
-
-console.log('Calling Travis...');
-
-
-got.post(`https://api.travis-ci.org/repo/climateimpactlab%2Fclawpack/requests`, {
-  headers: {
-    "Content-Type": "application/json",
-    "Accept": "application/json",
-    "Travis-API-Version": "3",
-    "Authorization": `token ${process.env.TRAVIS_API_TOKEN}`,
-  },
-  body: JSON.stringify({
-    request: {
-      message: `Trigger build at ClimateImpactLab/clawpack commit: ${gitCommitHash}`,
-      branch: 'source',
-    },    
-  }),
-})
-.then(() => {
-  console.log("Triggered build of ClimateImpactLab/clawpack");
-})
-.catch((err) => {
-  console.error(err);
-  
-  process.exit(-1);
-});
\ No newline at end of file

From 8b6c869e8a862a066843150cd1f56575187e05ca Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 09:28:28 -0700
Subject: [PATCH 082/224] update submod

---
 .gitmodules | 3 +++
 clawpack    | 1 +
 2 files changed, 4 insertions(+)
 create mode 100644 .gitmodules
 create mode 160000 clawpack

diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..65b4ba2
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "clawpack"]
+	path = clawpack
+	url = https://github.com/ClimateImpactLab/clawpack
diff --git a/clawpack b/clawpack
new file mode 160000
index 0000000..c854612
--- /dev/null
+++ b/clawpack
@@ -0,0 +1 @@
+Subproject commit c85461231106286f9ba037261bccfe1fb2f76bac

From d11d700ef70c7ad9f35d3cc2498f250b15e58f16 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 09:41:05 -0700
Subject: [PATCH 083/224] rm clawpack submod

---
 .gitmodules | 3 ---
 clawpack    | 1 -
 2 files changed, 4 deletions(-)
 delete mode 100644 .gitmodules
 delete mode 160000 clawpack

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 65b4ba2..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "clawpack"]
-	path = clawpack
-	url = https://github.com/ClimateImpactLab/clawpack
diff --git a/clawpack b/clawpack
deleted file mode 160000
index c854612..0000000
--- a/clawpack
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit c85461231106286f9ba037261bccfe1fb2f76bac

From d1cd3e30f3aecfe236fca112c8a583d36bede308 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 09:42:00 -0700
Subject: [PATCH 084/224] simplify clawpack install

---
 notebook/Dockerfile | 10 +---------
 worker/Dockerfile   | 12 +-----------
 2 files changed, 2 insertions(+), 20 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index ee642fc..9e4cfb4 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -77,25 +77,17 @@ RUN jupyter labextension update --all && \
 ## clawpack added for coastal-specific image
 ##################
 
-COPY ./clawpack /clawpack
-
-## install Clawpack
 ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
 
-WORKDIR /clawpack
-
-# need to change shell in order for source command to work
-SHELL ["/bin/bash", "-c"]
-
 # super sketchy hack to get around our need for compiler_compat binaries and some
 # other things that cause problems together?
 # see https://github.com/ContinuumIO/anaconda-issues/issues/11152
 RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
 
 # install clawpack
-RUN pip install -e .
+RUN pip install -e --user --src=/clawpack git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac
 
 ##################
 
diff --git a/worker/Dockerfile b/worker/Dockerfile
index e7e305a..f6f066e 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -58,27 +58,17 @@ ENV OPENBLAS_NUM_THREADS=1
 ## clawpack added for coastal-specific image
 ##################
 
-COPY ./clawpack /clawpack
-
-## install Clawpack
 ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
 
-WORKDIR /clawpack
-
-# need to change shell in order for source command to work
-SHELL ["/bin/bash", "-c"]
-
 # super sketchy hack to get around our need for compiler_compat binaries and some
 # other things that cause problems together?
 # see https://github.com/ContinuumIO/anaconda-issues/issues/11152
 RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
 
 # install clawpack
-RUN pip install -e .
-
-WORKDIR /
+RUN pip install -e --user --src=/clawpack git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac
 
 ##################
 

From 7097c2df86d646e9627ef0e5abaa95a0778a8e3b Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 09:49:29 -0700
Subject: [PATCH 085/224] undo cp clawpack

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7d40c60..c927d37 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,7 +18,6 @@ install:
 - "cat notebook/worker-template.yml | grep image:"
 - "cp base_environment.yml $IMAGE_NAME/base_environment.yml"
 - "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh"
-- "cp -r clawpack $IMAGE_NAME/clawpack"
 - "cd $IMAGE_NAME"
 
 

From f890e3d63c073b80682e368137d02fd3d23d5c54 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 10:32:32 -0700
Subject: [PATCH 086/224] try again w/ better pip install

---
 notebook/Dockerfile | 2 +-
 worker/Dockerfile   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 9e4cfb4..71f09f4 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -87,7 +87,7 @@ ENV FC=gfortran
 RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
 
 # install clawpack
-RUN pip install -e --user --src=/clawpack git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac
+RUN pip install --src=${CLAW} -e git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac
 
 ##################
 
diff --git a/worker/Dockerfile b/worker/Dockerfile
index f6f066e..95ef5a2 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -68,7 +68,7 @@ ENV FC=gfortran
 RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
 
 # install clawpack
-RUN pip install -e --user --src=/clawpack git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac
+RUN pip install --src=${CLAW}-e git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac
 
 ##################
 

From 3cfcb07dc70736319f0edf2a4c16c22616951893 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 10:41:21 -0700
Subject: [PATCH 087/224] specify egg in install

---
 notebook/Dockerfile | 2 +-
 worker/Dockerfile   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 71f09f4..89242cc 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -87,7 +87,7 @@ ENV FC=gfortran
 RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
 
 # install clawpack
-RUN pip install --src=${CLAW} -e git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac
+RUN pip install --src=${CLAW} -e git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac#egg=clawpack
 
 ##################
 
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 95ef5a2..9d1df30 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -68,7 +68,7 @@ ENV FC=gfortran
 RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
 
 # install clawpack
-RUN pip install --src=${CLAW}-e git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac
+RUN pip install --src=${CLAW} -e git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac#egg=clawpack
 
 ##################
 

From b97a2b8f999c22334b7207e3233b98467950b0a9 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 11:15:05 -0700
Subject: [PATCH 088/224] assign central clawpack commit

---
 .travis.yml         | 10 +++++++---
 notebook/Dockerfile | 12 +++++++-----
 worker/Dockerfile   | 12 +++++++-----
 3 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c927d37..74bbee2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,8 +2,11 @@ services:
   - docker
 
 env:
-- IMAGE_NAME=notebook
-- IMAGE_NAME=worker
+  jobs:
+    - IMAGE_NAME=notebook
+    - IMAGE_NAME=worker
+  global:
+    - CLAWPACK_COMMIT=c85461231106286f9ba037261bccfe1fb2f76bac
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
@@ -22,7 +25,8 @@ install:
 
 
 script:
-- docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT .
+- docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT \
+  --build-arg CLAWPACK_COMMIT .
 
 deploy:
 - provider: script
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 89242cc..b871a2f 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -77,17 +77,19 @@ RUN jupyter labextension update --all && \
 ## clawpack added for coastal-specific image
 ##################
 
+ARG CLAWPACK_COMMIT
 ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
 
-# super sketchy hack to get around our need for compiler_compat binaries and some
-# other things that cause problems together?
-# see https://github.com/ContinuumIO/anaconda-issues/issues/11152
-RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
+# need to change shell in order for source command to work
+SHELL ["/bin/bash", "-c"]
 
 # install clawpack
-RUN pip install --src=${CLAW} -e git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac#egg=clawpack
+RUN source /opt/conda/etc/profile.d/conda.sh \
+    && conda activate \
+    && pip install --src=$(dirname $CLAW) -e \
+      git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 
 ##################
 
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 9d1df30..dbb0c76 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -58,17 +58,19 @@ ENV OPENBLAS_NUM_THREADS=1
 ## clawpack added for coastal-specific image
 ##################
 
+ARG CLAWPACK_COMMIT
 ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
 
-# super sketchy hack to get around our need for compiler_compat binaries and some
-# other things that cause problems together?
-# see https://github.com/ContinuumIO/anaconda-issues/issues/11152
-RUN mv /opt/conda/compiler_compat/ld /opt/conda/compiler_compat/ld-conda
+# need to change shell in order for source command to work
+SHELL ["/bin/bash", "-c"]
 
 # install clawpack
-RUN pip install --src=${CLAW} -e git+https://github.com/climateimpactlab/clawpack.git@c85461231106286f9ba037261bccfe1fb2f76bac#egg=clawpack
+RUN source /opt/conda/etc/profile.d/conda.sh \
+    && conda activate \
+    && pip install --src=$(dirname $CLAW) -e \
+        git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 
 ##################
 

From fe86e5d54856f18319a06c421a80cf1d82284a87 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 11:28:06 -0700
Subject: [PATCH 089/224] fix build_arg in travis

---
 .travis.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 74bbee2..ff46ecb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,8 +25,7 @@ install:
 
 
 script:
-- docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT \
-  --build-arg CLAWPACK_COMMIT .
+- docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
 
 deploy:
 - provider: script

From 80c09952bd2fa38fd442e4abd006678b934747da Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 12:13:37 -0700
Subject: [PATCH 090/224] test the build as well

---
 .travis.yml                       | 1 +
 base_environment.yml              | 1 +
 notebook/notebook_environment.yml | 1 -
 3 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index ff46ecb..fcf1bb3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,6 +26,7 @@ install:
 
 script:
 - docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
+- docker run -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT bash /clawpack/run_tests.sh
 
 deploy:
 - provider: script
diff --git a/base_environment.yml b/base_environment.yml
index 9c97138..2e3eec1 100755
--- a/base_environment.yml
+++ b/base_environment.yml
@@ -42,6 +42,7 @@ dependencies:
   - nc-time-axis=1.2.0=py_1
   - netcdf-fortran=4.5.3=nompi_hfef6a68_100
   - netcdf4=1.5.3=nompi_py37hdc49583_105
+  - nose=1.3.7=py37hc8dfbb8_1004
   - numba=0.49.1=py37h0da4684_0
   - pandas=1.0.5=py37h0da4684_0
   - phantomjs=2.1.1=1 # for geoviews
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index e9a0191..b2594a0 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -15,7 +15,6 @@ dependencies:
   - jupyterlab_code_formatter=1.3.1=py_0
   - nano=2.9.8=h969982e_1000
   - nb_conda_kernels=2.2.3=py37_0
-  - nose=1.3.7=py37hc8dfbb8_1004
   - octave_kernel=0.32.0=pyh9f0ad1d_0
   - oct2py=5.0.4=py_0
   - openssh=8.3p1=h5957347_0

From 8a5ff042f12fc87daaac38c455ece6dd83691ab3 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 15:08:42 -0700
Subject: [PATCH 091/224] set ulimit in prepare

---
 notebook/prepare.sh | 3 +++
 worker/prepare.sh   | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/notebook/prepare.sh b/notebook/prepare.sh
index 115197e..c621744 100644
--- a/notebook/prepare.sh
+++ b/notebook/prepare.sh
@@ -34,5 +34,8 @@ if [ -f "/home/jovyan/worker-template.yml" ]; then
     python /home/jovyan/add_service_creds.py;
 fi
 
+# needed for CLAWPACK to not throw segfaults sometimes
+ulimit -s unlimited
+
 # Run extra commands
 $@
diff --git a/worker/prepare.sh b/worker/prepare.sh
index 90ebc0e..70acb9e 100644
--- a/worker/prepare.sh
+++ b/worker/prepare.sh
@@ -42,5 +42,8 @@ if [[ "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then
     gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS;
 fi
 
+# needed for CLAWPACK to not throw segfaults sometimes
+ulimit -s unlimited
+
 # Run extra commands
 $@

From 347a8595bc0bc94506b2b3ea0a65fd1a9eced7e3 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 17:02:56 -0700
Subject: [PATCH 092/224] update to new clawpack

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index fcf1bb3..4c7b064 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=c85461231106286f9ba037261bccfe1fb2f76bac
+    - CLAWPACK_COMMIT=6309956baeb39e8cf69e91037ed7820ca1cc0356
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From 9a210675e84a700485a933c34374e5b9b06b01c3 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 17:09:19 -0700
Subject: [PATCH 093/224] increase stack size for clawpack test

---
 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 4c7b064..2274c29 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,7 +26,8 @@ install:
 
 script:
 - docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
-- docker run -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT bash /clawpack/run_tests.sh
+- docker run --ulimit stack=8277716992:8277716992 \
+  -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT bash /clawpack/run_tests.sh
 
 deploy:
 - provider: script

From 9054f74e54c9d782461043c245892ebe05744048 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 17:23:45 -0700
Subject: [PATCH 094/224] fix docker run_tests in travis

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 2274c29..2020e89 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,7 +26,7 @@ install:
 
 script:
 - docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
-- docker run --ulimit stack=8277716992:8277716992 \
+- docker run --ulimit stack=8277716992:8277716992 
   -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT bash /clawpack/run_tests.sh
 
 deploy:

From 58f55914f54f689851b199cb2025bbdc7aca4ccb Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 17:32:12 -0700
Subject: [PATCH 095/224] update clawpack commit

---
 .travis.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2020e89..79a0678 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=6309956baeb39e8cf69e91037ed7820ca1cc0356
+    - CLAWPACK_COMMIT=bf4eefb3543d65331eb92848b2c0514a218194eb
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
@@ -26,8 +26,7 @@ install:
 
 script:
 - docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
-- docker run --ulimit stack=8277716992:8277716992 
-  -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT bash /clawpack/run_tests.sh
+- docker run -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT bash /clawpack/run_tests.sh
 
 deploy:
 - provider: script

From 28f8f5f3df82c61078b765b92601197a73e9b36f Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 17:50:03 -0700
Subject: [PATCH 096/224] set lib_paths for clawpack

---
 notebook/Dockerfile | 1 +
 worker/Dockerfile   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index b871a2f..4f33e2a 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -81,6 +81,7 @@ ARG CLAWPACK_COMMIT
 ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
+ENV LIB_PATHS=/opt/conda/lib
 
 # need to change shell in order for source command to work
 SHELL ["/bin/bash", "-c"]
diff --git a/worker/Dockerfile b/worker/Dockerfile
index dbb0c76..a3b0e52 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -62,6 +62,7 @@ ARG CLAWPACK_COMMIT
 ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
+ENV LIB_PATHS=/opt/conda/lib
 
 # need to change shell in order for source command to work
 SHELL ["/bin/bash", "-c"]

From a15451c35ef15771f82ff46d979f3462bb02a306 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 18:26:54 -0700
Subject: [PATCH 097/224] bump clawpack commit

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 79a0678..0af2cf0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=bf4eefb3543d65331eb92848b2c0514a218194eb
+    - CLAWPACK_COMMIT=6485e04a0daee160612e7fabd91c440835901903
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From 8b4df7726d59413adae1cf22ecd2a4d0e0e32a27 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 18:54:33 -0700
Subject: [PATCH 098/224] build xgeoclaw in base image

---
 .travis.yml         |  1 +
 Makefile.clawpack   | 58 +++++++++++++++++++++++++++++++++++++++++++++
 notebook/Dockerfile | 10 +++++++-
 worker/Dockerfile   | 28 ++++++++++++++--------
 4 files changed, 86 insertions(+), 11 deletions(-)
 create mode 100644 Makefile.clawpack

diff --git a/.travis.yml b/.travis.yml
index 0af2cf0..5a29786 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,6 +21,7 @@ install:
 - "cat notebook/worker-template.yml | grep image:"
 - "cp base_environment.yml $IMAGE_NAME/base_environment.yml"
 - "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh"
+- "cp Makefile.clawpack $IMAGE_NAME/Makefile.clawpack"
 - "cd $IMAGE_NAME"
 
 
diff --git a/Makefile.clawpack b/Makefile.clawpack
new file mode 100644
index 0000000..b0078fa
--- /dev/null
+++ b/Makefile.clawpack
@@ -0,0 +1,58 @@
+# Makefile for Clawpack code in this directory.
+# This version only sets the local files and frequently changed
+# options, and then includes the standard makefile pointed to by CLAWMAKE.
+CLAWMAKE ?= $(CLAW)/clawutil/src/Makefile.common
+
+# See the above file for details and a list of make options, or type
+#   make .help
+# at the unix prompt.
+
+
+# Adjust these variables if desired:
+# ----------------------------------
+
+CLAW_PKG = geoclaw                  # Clawpack package to use
+EXE ?= $(CLAW)/geoclaw/xgeoclaw     # Executable to create/use
+SETRUN_FILE = setrun.py        # File containing function to make data
+OUTDIR = _output               # Directory for output
+SETPLOT_FILE = setplot.py      # File containing function to set plots
+PLOTDIR = _plots               # Directory for plots
+
+# Environment variable FC should be set to fortran compiler, e.g. gfortran
+FFLAGS ?=
+
+# ---------------------------------
+# package sources for this program:
+# ---------------------------------
+
+GEOLIB = $(CLAW)/geoclaw/src/2d/shallow
+include $(GEOLIB)/Makefile.geoclaw
+
+# ---------------------------------------
+# package sources specifically to exclude
+# (i.e. if a custom replacement source
+#  under a different name is provided)
+# ---------------------------------------
+
+EXCLUDE_MODULES = \
+
+EXCLUDE_SOURCES = \
+
+# ----------------------------------------
+# List of custom sources for this program:
+# ----------------------------------------
+
+RIEMANN = $(CLAW)/riemann/src
+
+MODULES = \
+
+SOURCES = \
+  $(RIEMANN)/rpn2_geoclaw.f \
+  $(RIEMANN)/rpt2_geoclaw.f \
+  $(RIEMANN)/geoclaw_riemann_utils.f \
+
+#-------------------------------------------------------------------
+# Include Makefile containing standard definitions and make options:
+include $(CLAWMAKE)
+
+### DO NOT remove this line - make depends on it ###
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 4f33e2a..28149df 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -25,6 +25,7 @@ COPY config.yaml /pre-home
 
 RUN sudo mkdir /tempdir
 COPY common.sh /tempdir
+COPY Makefile.clawpack /tempdir
 
 COPY prepare.sh /usr/bin
 COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json
@@ -92,13 +93,20 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
     && pip install --src=$(dirname $CLAW) -e \
       git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 
+RUN export EXE=$CLAW/geoclaw/xgeoclaw && \
+  export FFLAGS="-DNETCDF -O3 -funroll-loops -finline-functions \
+      -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include" && \
+  make -f /tempdir/Makefile.clawpack new && \
+  export EXE=$CLAW/geoclaw/xgeoclaw_omp && \
+  export FFLAGS="$FFLAGS -fopenmp" && \
+  make -f /tempdir/Makefile.clawpack new
+
 ##################
 
 ## clean up
 RUN sudo rm -rf /var/lib/apt/lists/* /tempdir
 RUN conda clean --all -f
 
-
 ## prepare container
 WORKDIR $HOME
 ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
diff --git a/worker/Dockerfile b/worker/Dockerfile
index a3b0e52..4572b60 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -7,6 +7,7 @@ ENV MPLBACKEND=Agg
 ## filepath curation
 RUN mkdir /tempdir
 COPY common.sh /tempdir
+COPY Makefile.clawpack /tempdir
 
 COPY add_service_creds.py /usr/bin
 COPY prepare.sh /usr/bin
@@ -44,16 +45,6 @@ RUN conda update -n base conda
 RUN conda env update -f /opt/conda/specs/base_environment.yml
 RUN conda list -n base
 
-## clean up
-RUN rm -rf /var/lib/apt/lists/* /tempdir
-RUN conda clean --all -f
-
-
-## prepare container
-ENV OMP_NUM_THREADS=1
-ENV MKL_NUM_THREADS=1
-ENV OPENBLAS_NUM_THREADS=1
-
 ##################
 ## clawpack added for coastal-specific image
 ##################
@@ -73,6 +64,23 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
     && pip install --src=$(dirname $CLAW) -e \
         git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 
+RUN export EXE=$CLAW/geoclaw/xgeoclaw && \
+  export FFLAGS="-DNETCDF -O3 -funroll-loops -finline-functions \
+      -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include" && \
+  make -f /tempdir/Makefile.clawpack new && \
+  export EXE=$CLAW/geoclaw/xgeoclaw_omp && \
+  export FFLAGS="$FFLAGS -fopenmp" && \
+  make -f /tempdir/Makefile.clawpack new
+
 ##################
 
+## clean up
+RUN rm -rf /var/lib/apt/lists/* /tempdir
+RUN conda clean --all -f
+
+## prepare container
+ENV OMP_NUM_THREADS=1
+ENV MKL_NUM_THREADS=1
+ENV OPENBLAS_NUM_THREADS=1
+
 ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]

From 5eb3e63327aa36a5f7ef9d53999857c089118abf Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 21:35:25 -0700
Subject: [PATCH 099/224] sudo pip install clawpack

---
 notebook/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 28149df..1ed2c8a 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -90,7 +90,7 @@ SHELL ["/bin/bash", "-c"]
 # install clawpack
 RUN source /opt/conda/etc/profile.d/conda.sh \
     && conda activate \
-    && pip install --src=$(dirname $CLAW) -e \
+    && sudo pip install --src=$(dirname $CLAW) -e \
       git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 
 RUN export EXE=$CLAW/geoclaw/xgeoclaw && \

From 291a434448614ad85dd31628ea4f6aba00377aaf Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 6 Jul 2020 22:38:04 -0700
Subject: [PATCH 100/224] try root user for pip install

---
 notebook/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 1ed2c8a..eb29087 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -87,11 +87,13 @@ ENV LIB_PATHS=/opt/conda/lib
 # need to change shell in order for source command to work
 SHELL ["/bin/bash", "-c"]
 
-# install clawpack
+# install clawpack (use root to access /clawpack)
+USER root
 RUN source /opt/conda/etc/profile.d/conda.sh \
     && conda activate \
     && sudo pip install --src=$(dirname $CLAW) -e \
       git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
+USER $NB_USER
 
 RUN export EXE=$CLAW/geoclaw/xgeoclaw && \
   export FFLAGS="-DNETCDF -O3 -funroll-loops -finline-functions \

From 91ea8df89a619dfb00137d73a1324595632a8872 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 7 Jul 2020 06:13:02 -0700
Subject: [PATCH 101/224] no sudo for root pip install

---
 notebook/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index eb29087..9b412c9 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -91,7 +91,7 @@ SHELL ["/bin/bash", "-c"]
 USER root
 RUN source /opt/conda/etc/profile.d/conda.sh \
     && conda activate \
-    && sudo pip install --src=$(dirname $CLAW) -e \
+    && pip install --src=$(dirname $CLAW) -e \
       git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 USER $NB_USER
 

From 3b1ee82f8c582e3c596467872a09f1875b5c24aa Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 7 Jul 2020 07:07:03 -0700
Subject: [PATCH 102/224] pin notebook sha and add fix-permissions

---
 .travis.yml                                   |  4 +-
 notebook/Dockerfile                           |  5 ++-
 .../Makefile.clawpack                         |  0
 .../base_environment.yml                      |  0
 common.sh => shared_resources/common.sh       |  0
 shared_resources/fix-permissions.sh           | 39 +++++++++++++++++++
 worker/Dockerfile                             |  2 +-
 7 files changed, 44 insertions(+), 6 deletions(-)
 rename Makefile.clawpack => shared_resources/Makefile.clawpack (100%)
 rename base_environment.yml => shared_resources/base_environment.yml (100%)
 rename common.sh => shared_resources/common.sh (100%)
 create mode 100644 shared_resources/fix-permissions.sh

diff --git a/.travis.yml b/.travis.yml
index 5a29786..be9fb2e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,9 +19,7 @@ install:
       notebook/worker-template.yml; fi"
 - "rm notebook/worker-template.yml.bak"
 - "cat notebook/worker-template.yml | grep image:"
-- "cp base_environment.yml $IMAGE_NAME/base_environment.yml"
-- "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh"
-- "cp Makefile.clawpack $IMAGE_NAME/Makefile.clawpack"
+- "cp shared_resources/* $IMAGE_NAME && chmod -R +x $IMAGE_NAME"
 - "cd $IMAGE_NAME"
 
 
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 9b412c9..d44ecf5 100644
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook:notebook-6.0.3
+FROM jupyter/base-notebook@sha256:135779f39abfdfe75a6dbc50c7a15c55cc79c8231cd75f83aaabf87127c60adf
 ARG DEBIAN_FRONTEND=noninteractive
 
 ## needed to make sure things with cython compile correctly
@@ -91,9 +91,10 @@ SHELL ["/bin/bash", "-c"]
 USER root
 RUN source /opt/conda/etc/profile.d/conda.sh \
     && conda activate \
-    && pip install --src=$(dirname $CLAW) -e \
+    && sudo pip install --src=$(dirname $CLAW) -e \
       git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 USER $NB_USER
+RUN fix-permissions.sh /clawpack
 
 RUN export EXE=$CLAW/geoclaw/xgeoclaw && \
   export FFLAGS="-DNETCDF -O3 -funroll-loops -finline-functions \
diff --git a/Makefile.clawpack b/shared_resources/Makefile.clawpack
similarity index 100%
rename from Makefile.clawpack
rename to shared_resources/Makefile.clawpack
diff --git a/base_environment.yml b/shared_resources/base_environment.yml
similarity index 100%
rename from base_environment.yml
rename to shared_resources/base_environment.yml
diff --git a/common.sh b/shared_resources/common.sh
similarity index 100%
rename from common.sh
rename to shared_resources/common.sh
diff --git a/shared_resources/fix-permissions.sh b/shared_resources/fix-permissions.sh
new file mode 100644
index 0000000..c8d0032
--- /dev/null
+++ b/shared_resources/fix-permissions.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+## Adapted from:
+## https://github.com/jupyter/docker-stacks/blob/master/base-notebook/fix-permissions
+
+# set permissions on a directory
+# after any installation, if a directory needs to be (human) user-writable,
+# run this script on it.
+# It will make everything in the directory owned by the group $NB_GID
+# and writable by that group.
+# Deployments that want to set a specific user id can preserve permissions
+# by adding the `--group-add users` line to `docker run`.
+
+# uses find to avoid touching files that already have the right permissions,
+# which would cause massive image explosion
+
+# right permissions are:
+# group=$NB_GID
+# AND permissions include group rwX (directory-execute)
+# AND directories have setuid,setgid bits set
+
+set -e
+
+for d in "$@"; do
+  find "$d" \
+    ! \( \
+      -group $NB_GID \
+      -a -perm -g+rwX  \
+    \) \
+    -exec chgrp $NB_GID {} \; \
+    -exec chmod g+rwX {} \;
+  # setuid,setgid *on directories only*
+  find "$d" \
+    \( \
+        -type d \
+        -a ! -perm -6000  \
+    \) \
+    -exec chmod +6000 {} \;
+done
\ No newline at end of file
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 4572b60..58c9884 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -1,5 +1,5 @@
 ## using same container as jupyter/base-notebook:python-3.7.3
-FROM ubuntu:bionic-20200403@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
+FROM ubuntu:focal-20200423@sha256:238e696992ba9913d24cfc3727034985abd136e08ee3067982401acdc30cbf3f
 ARG DEBIAN_FRONTEND=noninteractive
 ENV NPY_DISTUTILS_APPEND_FLAGS=1
 ENV MPLBACKEND=Agg

From bb05dd6eea185bd0a7ded3adc98fab6e95bf0ea5 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 7 Jul 2020 07:11:54 -0700
Subject: [PATCH 103/224] added executable permissions

---
 notebook/Dockerfile                 | 0
 notebook/add_service_creds.py       | 0
 notebook/config.yaml                | 0
 notebook/overrides.json             | 0
 notebook/prepare.sh                 | 0
 notebook/run_sql_proxy.py           | 0
 notebook/worker-template.yml        | 0
 shared_resources/Makefile.clawpack  | 0
 shared_resources/common.sh          | 0
 shared_resources/fix-permissions.sh | 0
 10 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 notebook/Dockerfile
 mode change 100644 => 100755 notebook/add_service_creds.py
 mode change 100644 => 100755 notebook/config.yaml
 mode change 100644 => 100755 notebook/overrides.json
 mode change 100644 => 100755 notebook/prepare.sh
 mode change 100644 => 100755 notebook/run_sql_proxy.py
 mode change 100644 => 100755 notebook/worker-template.yml
 mode change 100644 => 100755 shared_resources/Makefile.clawpack
 mode change 100644 => 100755 shared_resources/common.sh
 mode change 100644 => 100755 shared_resources/fix-permissions.sh

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
old mode 100644
new mode 100755
diff --git a/notebook/add_service_creds.py b/notebook/add_service_creds.py
old mode 100644
new mode 100755
diff --git a/notebook/config.yaml b/notebook/config.yaml
old mode 100644
new mode 100755
diff --git a/notebook/overrides.json b/notebook/overrides.json
old mode 100644
new mode 100755
diff --git a/notebook/prepare.sh b/notebook/prepare.sh
old mode 100644
new mode 100755
diff --git a/notebook/run_sql_proxy.py b/notebook/run_sql_proxy.py
old mode 100644
new mode 100755
diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml
old mode 100644
new mode 100755
diff --git a/shared_resources/Makefile.clawpack b/shared_resources/Makefile.clawpack
old mode 100644
new mode 100755
diff --git a/shared_resources/common.sh b/shared_resources/common.sh
old mode 100644
new mode 100755
diff --git a/shared_resources/fix-permissions.sh b/shared_resources/fix-permissions.sh
old mode 100644
new mode 100755

From 32ecc9eadc2fed20c8ed93479e7eea1a46c7adc7 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 7 Jul 2020 08:24:25 -0700
Subject: [PATCH 104/224] unsudo pip install

---
 notebook/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index d44ecf5..b8ac6f0 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -91,7 +91,7 @@ SHELL ["/bin/bash", "-c"]
 USER root
 RUN source /opt/conda/etc/profile.d/conda.sh \
     && conda activate \
-    && sudo pip install --src=$(dirname $CLAW) -e \
+    && pip install --src=$(dirname $CLAW) -e \
       git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 USER $NB_USER
 RUN fix-permissions.sh /clawpack

From 4d9ab7e84a3b0c739236466a098a1597eab7c7fe Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 7 Jul 2020 08:56:55 -0700
Subject: [PATCH 105/224] danced around permission issues for /clawpack

---
 notebook/Dockerfile | 7 ++++---
 worker/Dockerfile   | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index b8ac6f0..730b30d 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -26,6 +26,7 @@ COPY config.yaml /pre-home
 RUN sudo mkdir /tempdir
 COPY common.sh /tempdir
 COPY Makefile.clawpack /tempdir
+COPY fix-permissions.sh /tempdir
 
 COPY prepare.sh /usr/bin
 COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json
@@ -87,14 +88,14 @@ ENV LIB_PATHS=/opt/conda/lib
 # need to change shell in order for source command to work
 SHELL ["/bin/bash", "-c"]
 
-# install clawpack (use root to access /clawpack)
+# install clawpack (use root to access /clawpack and then fix permissions)
 USER root
 RUN source /opt/conda/etc/profile.d/conda.sh \
     && conda activate \
     && pip install --src=$(dirname $CLAW) -e \
       git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
+RUN /tempdir/fix-permissions.sh /clawpack
 USER $NB_USER
-RUN fix-permissions.sh /clawpack
 
 RUN export EXE=$CLAW/geoclaw/xgeoclaw && \
   export FFLAGS="-DNETCDF -O3 -funroll-loops -finline-functions \
@@ -108,7 +109,7 @@ RUN export EXE=$CLAW/geoclaw/xgeoclaw && \
 
 ## clean up
 RUN sudo rm -rf /var/lib/apt/lists/* /tempdir
-RUN conda clean --all -f
+RUN conda clean -yaf
 
 ## prepare container
 WORKDIR $HOME
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 58c9884..f879e6a 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -76,7 +76,7 @@ RUN export EXE=$CLAW/geoclaw/xgeoclaw && \
 
 ## clean up
 RUN rm -rf /var/lib/apt/lists/* /tempdir
-RUN conda clean --all -f
+RUN conda clean -yaf
 
 ## prepare container
 ENV OMP_NUM_THREADS=1

From a16dbb51a7dd5521601196029832c7d954dff344 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 7 Jul 2020 13:33:09 -0700
Subject: [PATCH 106/224] try newer start command from jupyter/base-notebook

---
 notebook/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 730b30d..3c20878 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -86,7 +86,7 @@ ENV FC=gfortran
 ENV LIB_PATHS=/opt/conda/lib
 
 # need to change shell in order for source command to work
-SHELL ["/bin/bash", "-c"]
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # install clawpack (use root to access /clawpack and then fix permissions)
 USER root
@@ -114,4 +114,4 @@ RUN conda clean -yaf
 ## prepare container
 WORKDIR $HOME
 ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
-CMD ["start.sh jupyter lab"]
+CMD ["start-notebook.sh"]

From e2d91d5e2aca57521a321ea364ffb2db46eeaab4 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 7 Jul 2020 14:18:57 -0700
Subject: [PATCH 107/224] add python-snappy for parquet compression

---
 shared_resources/base_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 2e3eec1..143656a 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -52,6 +52,7 @@ dependencies:
   - pyinterp=0.3.2=mkl_py37he448baa_102
   - pyshp=2.1.0=py_0
   - python=3.7.6=cpython_h8356626_6
+  - python-snappy=0.5.4=py37h7cfaab3_1
   - pyyaml=5.3.1=py37h8f50634_0
   - rasterio=1.1.5=py37h0492a4a_0
   - regionmask=0.5.0=py_1

From dbbfedb9fb9138e29909c2d0afdd24979b48a227 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Wed, 8 Jul 2020 06:57:15 -0700
Subject: [PATCH 108/224] fix perm for /opt/conda to allow user installs

---
 notebook/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 3c20878..7a61703 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -95,6 +95,7 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
     && pip install --src=$(dirname $CLAW) -e \
       git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 RUN /tempdir/fix-permissions.sh /clawpack
+RUN /tempdir/fix-permissions.sh /opt/conda
 USER $NB_USER
 
 RUN export EXE=$CLAW/geoclaw/xgeoclaw && \

From 0b5d1f5ac701a75a47e63183438b7ddcc27f7353 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 06:08:26 -0700
Subject: [PATCH 109/224] update claw fflags

---
 notebook/Dockerfile | 57 ++++++++++++++++++++++++++++++++++++++-------
 worker/Dockerfile   | 55 ++++++++++++++++++++++++++++++++++++-------
 2 files changed, 95 insertions(+), 17 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 7a61703..2a72135 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -84,6 +84,44 @@ ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
 ENV LIB_PATHS=/opt/conda/lib
+# Was getting weird behavior with any broad optimization flags (-O, -O2, -O3) wherein
+# the boundary velocity error might be triggered on different boundaries if you piped
+# both stdout and stderr to a file. If you only piped one or didn't pipe any, you 
+# got one answer and if you piped both you got a different answer. Without the flags,
+# you would get the same answer regardless (the one originally obtained when piping
+# both out and err). To maintain a level of optimization, I just took all of the flags
+# from O3 here: https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html and put them
+# in individually. A few were removed due to compile or runtime errors. In addition, I
+# added some flags applied by default from the conda gfortran compiler (-ftree-vectorize 
+# -fPIC -fno-plt -ffunction-sections). I also added the march flag for the google 
+# machines we are using (broadwell currently), and dropped fstack-protector-strong from
+# the list of conda-based flags, as it was throwing segfaults.
+ENV CLAW_FFLAGS="-DNETCDF -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include \
+  -fauto-inc-dec -fbranch-count-reg -fcombine-stack-adjustments -fcompare-elim \
+  -fcprop-registers -fdce -fdefer-pop -fdelayed-branch -fdse -fforward-propagate \
+  -fguess-branch-probability -fif-conversion -fif-conversion2 \
+  -finline-functions-called-once -fipa-profile -fipa-pure-const -fipa-reference \
+  -fmerge-constants -fmove-loop-invariants -fomit-frame-pointer -freorder-blocks \
+  -fshrink-wrap -fshrink-wrap-separate -fsplit-wide-types -fssa-backprop -fssa-phiopt \
+  -ftree-bit-ccp -ftree-ccp -ftree-ch -ftree-coalesce-vars -ftree-copy-prop -ftree-dce \
+  -ftree-dominator-opts -ftree-dse -ftree-forwprop -ftree-fre -ftree-phiprop \
+  -ftree-pta -ftree-scev-cprop -ftree-sink -ftree-slsr -ftree-sra -ftree-ter \
+  -funit-at-a-time -freorder-blocks-algorithm=stc -freorder-blocks-and-partition \
+  -freorder-functions -frerun-cse-after-loop -fschedule-insns -fschedule-insns2 \
+  -fsched-interblock -fsched-spec -fstore-merging -fstrict-aliasing -fthread-jumps \
+  -ftree-builtin-call-dce -ftree-pre -ftree-switch-conversion  -ftree-tail-merge \
+  -ftree-vrp -fgcse  -fgcse-lm  -fhoist-adjacent-loads -finline-functions \
+  -finline-small-functions -findirect-inlining -fipa-bit-cp  -fipa-cp  -fipa-icf \
+  -fipa-ra  -fipa-sra  -fipa-vrp -fisolate-erroneous-paths-dereference -flra-remat \
+  -foptimize-sibling-calls -foptimize-strlen -fpartial-inlining -fpeephole2 \
+  -falign-functions -falign-jumps -falign-labels  -falign-loops -fcaller-saves \
+  -fcode-hoisting -fcrossjumping -fcse-follow-jumps  -fcse-skip-blocks \
+  -fdelete-null-pointer-checks -fdevirtualize  -fdevirtualize-speculatively \
+  -fexpensive-optimizations -fgcse-after-reload -floop-unroll-and-jam -fpeel-loops \
+  -fpredictive-commoning -fsplit-loops -fsplit-paths -ftree-loop-distribution \
+  -ftree-loop-vectorize -ftree-partial-pre -ftree-slp-vectorize -funswitch-loops \
+  -fvect-cost-model -fvect-cost-model=dynamic -march=broadwell -ftree-vectorize -fPIC \
+  -fno-plt -ffunction-sections"
 
 # need to change shell in order for source command to work
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -91,19 +129,20 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 # install clawpack (use root to access /clawpack and then fix permissions)
 USER root
 RUN source /opt/conda/etc/profile.d/conda.sh \
-    && conda activate \
-    && pip install --src=$(dirname $CLAW) -e \
-      git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
+  && conda activate \
+  && pip install --src=$(dirname $CLAW) -e \
+    git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 RUN /tempdir/fix-permissions.sh /clawpack
 RUN /tempdir/fix-permissions.sh /opt/conda
 USER $NB_USER
 
-RUN export EXE=$CLAW/geoclaw/xgeoclaw && \
-  export FFLAGS="-DNETCDF -O3 -funroll-loops -finline-functions \
-      -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include" && \
-  make -f /tempdir/Makefile.clawpack new && \
-  export EXE=$CLAW/geoclaw/xgeoclaw_omp && \
-  export FFLAGS="$FFLAGS -fopenmp" && \
+RUN source /opt/conda/etc/profile.d/conda.sh \
+  && conda activate \
+  && export EXE=$CLAW/geoclaw/xgeoclaw \
+  && export FFLAGS=$CLAW_FFLAGS \
+  && make -f /tempdir/Makefile.clawpack new \
+  && export EXE=$CLAW/geoclaw/xgeoclaw_omp \
+  && export FFLAGS="$FFLAGS -fopenmp" \
   make -f /tempdir/Makefile.clawpack new
 
 ##################
diff --git a/worker/Dockerfile b/worker/Dockerfile
index f879e6a..8b96976 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -54,9 +54,47 @@ ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
 ENV LIB_PATHS=/opt/conda/lib
+# Was getting weird behavior with any broad optimization flags (-O, -O2, -O3) wherein
+# the boundary velocity error might be triggered on different boundaries if you piped
+# both stdout and stderr to a file. If you only piped one or didn't pipe any, you 
+# got one answer and if you piped both you got a different answer. Without the flags,
+# you would get the same answer regardless (the one originally obtained when piping
+# both out and err). To maintain a level of optimization, I just took all of the flags
+# from O3 here: https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html and put them
+# in individually. A few were removed due to compile or runtime errors. In addition, I
+# added some flags applied by default from the conda gfortran compiler (-ftree-vectorize 
+# -fPIC -fno-plt -ffunction-sections). I also added the march flag for the google 
+# machines we are using (broadwell currently), and dropped fstack-protector-strong from
+# the list of conda-based flags, as it was throwing segfaults.
+ENV CLAW_FFLAGS="-DNETCDF -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include \
+  -fauto-inc-dec -fbranch-count-reg -fcombine-stack-adjustments -fcompare-elim \
+  -fcprop-registers -fdce -fdefer-pop -fdelayed-branch -fdse -fforward-propagate \
+  -fguess-branch-probability -fif-conversion -fif-conversion2 \
+  -finline-functions-called-once -fipa-profile -fipa-pure-const -fipa-reference \
+  -fmerge-constants -fmove-loop-invariants -fomit-frame-pointer -freorder-blocks \
+  -fshrink-wrap -fshrink-wrap-separate -fsplit-wide-types -fssa-backprop -fssa-phiopt \
+  -ftree-bit-ccp -ftree-ccp -ftree-ch -ftree-coalesce-vars -ftree-copy-prop -ftree-dce \
+  -ftree-dominator-opts -ftree-dse -ftree-forwprop -ftree-fre -ftree-phiprop \
+  -ftree-pta -ftree-scev-cprop -ftree-sink -ftree-slsr -ftree-sra -ftree-ter \
+  -funit-at-a-time -freorder-blocks-algorithm=stc -freorder-blocks-and-partition \
+  -freorder-functions -frerun-cse-after-loop -fschedule-insns -fschedule-insns2 \
+  -fsched-interblock -fsched-spec -fstore-merging -fstrict-aliasing -fthread-jumps \
+  -ftree-builtin-call-dce -ftree-pre -ftree-switch-conversion  -ftree-tail-merge \
+  -ftree-vrp -fgcse  -fgcse-lm  -fhoist-adjacent-loads -finline-functions \
+  -finline-small-functions -findirect-inlining -fipa-bit-cp  -fipa-cp  -fipa-icf \
+  -fipa-ra  -fipa-sra  -fipa-vrp -fisolate-erroneous-paths-dereference -flra-remat \
+  -foptimize-sibling-calls -foptimize-strlen -fpartial-inlining -fpeephole2 \
+  -falign-functions -falign-jumps -falign-labels  -falign-loops -fcaller-saves \
+  -fcode-hoisting -fcrossjumping -fcse-follow-jumps  -fcse-skip-blocks \
+  -fdelete-null-pointer-checks -fdevirtualize  -fdevirtualize-speculatively \
+  -fexpensive-optimizations -fgcse-after-reload -floop-unroll-and-jam -fpeel-loops \
+  -fpredictive-commoning -fsplit-loops -fsplit-paths -ftree-loop-distribution \
+  -ftree-loop-vectorize -ftree-partial-pre -ftree-slp-vectorize -funswitch-loops \
+  -fvect-cost-model -fvect-cost-model=dynamic -march=broadwell -ftree-vectorize -fPIC \
+  -fno-plt -ffunction-sections"
 
 # need to change shell in order for source command to work
-SHELL ["/bin/bash", "-c"]
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # install clawpack
 RUN source /opt/conda/etc/profile.d/conda.sh \
@@ -64,13 +102,14 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
     && pip install --src=$(dirname $CLAW) -e \
         git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
 
-RUN export EXE=$CLAW/geoclaw/xgeoclaw && \
-  export FFLAGS="-DNETCDF -O3 -funroll-loops -finline-functions \
-      -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include" && \
-  make -f /tempdir/Makefile.clawpack new && \
-  export EXE=$CLAW/geoclaw/xgeoclaw_omp && \
-  export FFLAGS="$FFLAGS -fopenmp" && \
-  make -f /tempdir/Makefile.clawpack new
+RUN source /opt/conda/etc/profile.d/conda.sh \
+    && conda activate \
+    && export EXE=$CLAW/geoclaw/xgeoclaw \
+    && export FFLAGS=$CLAW_FFLAGS \
+    && make -f /tempdir/Makefile.clawpack new \
+    && export EXE=$CLAW/geoclaw/xgeoclaw_omp \
+    && export FFLAGS="$FFLAGS -fopenmp" \
+    && make -f /tempdir/Makefile.clawpack new
 
 ##################
 

From 15277492ea75ec03fdc381af254edb369654a797 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 06:12:45 -0700
Subject: [PATCH 110/224] add octave-optim

---
 notebook/Dockerfile      | 4 ++--
 octave-worker/Dockerfile | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 2a72135..cc55df2 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -38,9 +38,9 @@ RUN sudo chown -R $NB_USER /gcs
 
 
 ## more apt-get
-RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
   vim less man locate kmod dialog \
-  octave && \
+  octave octave-optim && \
   sudo apt-get clean
 
 
diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile
index 4ac3655..2879983 100644
--- a/octave-worker/Dockerfile
+++ b/octave-worker/Dockerfile
@@ -6,7 +6,8 @@ ARG DEBIAN_FRONTEND=noninteractive
 COPY octave_environment.yml /opt/conda/specs/octave_environment.yml
 
 ## install octave
-RUN sudo DEBIAN_FRONTEND=noninteractive apt-get update && sudo apt-get install -yq octave
+RUN sudo DEBIAN_FRONTEND=noninteractive apt-get update \
+    && sudo apt-get install -yq --no-install-recommends octave octave-optim
 
 # add octave-specific packages
 RUN conda env update -f /opt/conda/specs/octave_environment.yml

From bf0e81ee43450501dbcb5a0acdbe908b55e13da7 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 06:24:07 -0700
Subject: [PATCH 111/224] fix regex for octave-worker

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index be9fb2e..b28f5eb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -71,7 +71,7 @@ deploy:
     docker push "rhodium/octave-worker:$TRAVIS_BRANCH"
   on:
     all_branches: true
-    condition: ("$TRAVIS_BRANCH" =~ "^dev") && ("$IMAGE_NAME" == "worker")
+    condition: ($TRAVIS_BRANCH =~ ^dev) && ($IMAGE_NAME == worker)
 
 - provider: script
   script: >-

From a6a4f72fefec56a97ba91c86ef2f05d7dddc2433 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 06:46:25 -0700
Subject: [PATCH 112/224] drop delayed-branch flag b/c not supported

---
 notebook/Dockerfile | 2 +-
 worker/Dockerfile   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index cc55df2..a0e4627 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -98,7 +98,7 @@ ENV LIB_PATHS=/opt/conda/lib
 # the list of conda-based flags, as it was throwing segfaults.
 ENV CLAW_FFLAGS="-DNETCDF -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include \
   -fauto-inc-dec -fbranch-count-reg -fcombine-stack-adjustments -fcompare-elim \
-  -fcprop-registers -fdce -fdefer-pop -fdelayed-branch -fdse -fforward-propagate \
+  -fcprop-registers -fdce -fdefer-pop -fdse -fforward-propagate \
   -fguess-branch-probability -fif-conversion -fif-conversion2 \
   -finline-functions-called-once -fipa-profile -fipa-pure-const -fipa-reference \
   -fmerge-constants -fmove-loop-invariants -fomit-frame-pointer -freorder-blocks \
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 8b96976..37855d1 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -68,7 +68,7 @@ ENV LIB_PATHS=/opt/conda/lib
 # the list of conda-based flags, as it was throwing segfaults.
 ENV CLAW_FFLAGS="-DNETCDF -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include \
   -fauto-inc-dec -fbranch-count-reg -fcombine-stack-adjustments -fcompare-elim \
-  -fcprop-registers -fdce -fdefer-pop -fdelayed-branch -fdse -fforward-propagate \
+  -fcprop-registers -fdce -fdefer-pop -fdse -fforward-propagate \
   -fguess-branch-probability -fif-conversion -fif-conversion2 \
   -finline-functions-called-once -fipa-profile -fipa-pure-const -fipa-reference \
   -fmerge-constants -fmove-loop-invariants -fomit-frame-pointer -freorder-blocks \

From d3b5337ac352aac2a3bc5a3536a55be85c593680 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 06:48:07 -0700
Subject: [PATCH 113/224] fix make line in nb dockerfile

---
 notebook/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index a0e4627..c19effc 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -143,7 +143,7 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
   && make -f /tempdir/Makefile.clawpack new \
   && export EXE=$CLAW/geoclaw/xgeoclaw_omp \
   && export FFLAGS="$FFLAGS -fopenmp" \
-  make -f /tempdir/Makefile.clawpack new
+  && make -f /tempdir/Makefile.clawpack new
 
 ##################
 

From b9cf0b2ea693bfa92ce643ae3ef357195c5a3a1d Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 07:06:02 -0700
Subject: [PATCH 114/224] no sudo in octave-worker

---
 octave-worker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile
index 2879983..5e16bde 100644
--- a/octave-worker/Dockerfile
+++ b/octave-worker/Dockerfile
@@ -6,8 +6,8 @@ ARG DEBIAN_FRONTEND=noninteractive
 COPY octave_environment.yml /opt/conda/specs/octave_environment.yml
 
 ## install octave
-RUN sudo DEBIAN_FRONTEND=noninteractive apt-get update \
-    && sudo apt-get install -yq --no-install-recommends octave octave-optim
+RUN apt-get update \
+    && apt-get install -yq --no-install-recommends octave octave-optim
 
 # add octave-specific packages
 RUN conda env update -f /opt/conda/specs/octave_environment.yml

From 55567861754bc78cc9fa9fb33e246972b4160a87 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 09:35:07 -0700
Subject: [PATCH 115/224] update jlab code formatter

---
 notebook/notebook_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index b2594a0..f0e6b9c 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -12,7 +12,7 @@ dependencies:
   - ipyleaflet=0.13.1=pyh9f0ad1d_0
   - ipympl=0.5.6=pyh9f0ad1d_1
   - isort=4.3.21=py37hc8dfbb8_1
-  - jupyterlab_code_formatter=1.3.1=py_0
+  - jupyterlab_code_formatter=1.3.2=py_0
   - nano=2.9.8=h969982e_1000
   - nb_conda_kernels=2.2.3=py37_0
   - octave_kernel=0.32.0=pyh9f0ad1d_0

From 30feda5cd65d47634f645673de6cebfb79d9e55d Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 09:36:52 -0700
Subject: [PATCH 116/224] unpin notebook packages

---
 notebook/notebook_environment.yml | 44 +++++++++++++++----------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index f0e6b9c..99f28f6 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -2,29 +2,29 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - black=19.10b0=py37_0
-  - bump2version=1.0.0=pyh9f0ad1d_0
-  - coverage=5.1=py37h8f50634_0
-  - dask-labextension=2.0.2=py_0
-  - flake8=3.8.3=pyh9f0ad1d_0
-  - gitpython=3.1.3=py_0
-  - ipdb=0.13.0=pyh9f0ad1d_0
-  - ipyleaflet=0.13.1=pyh9f0ad1d_0
-  - ipympl=0.5.6=pyh9f0ad1d_1
-  - isort=4.3.21=py37hc8dfbb8_1
-  - jupyterlab_code_formatter=1.3.2=py_0
-  - nano=2.9.8=h969982e_1000
-  - nb_conda_kernels=2.2.3=py37_0
+  - black
+  - bump2version
+  - coverage
+  - dask-labextension
+  - flake8
+  - gitpython
+  - ipdb
+  - ipyleaflet
+  - ipympl
+  - isort
+  - jupyterlab_code_formatter
+  - nano
+  - nb_conda_kernels
   - octave_kernel=0.32.0=pyh9f0ad1d_0
   - oct2py=5.0.4=py_0
-  - openssh=8.3p1=h5957347_0
-  - pip=20.1.1=py_1
-  - pytest=5.4.3=py37hc8dfbb8_0
-  - pytest-cov=2.10.0=pyh9f0ad1d_0
-  - pytest-runner=5.2=py_0
-  - python-graphviz=0.14=pyh9f0ad1d_0
-  - sidecar=0.4.0=pyh9f0ad1d_0
-  - sphinx=3.1.1=py_0
-  - tox=3.16.1=py37hc8dfbb8_0
+  - openssh
+  - pip
+  - pytest
+  - pytest-cov
+  - pytest-runner
+  - python-graphviz
+  - sidecar
+  - sphinx
+  - tox
   - pip:
     - black_nbconvert

From 6507e9278d681d5efd4195704f3d16df7f591443 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 10:32:58 -0700
Subject: [PATCH 117/224] pin jlab code formatter to match extension to pkg

---
 notebook/Dockerfile               | 2 +-
 notebook/notebook_environment.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index c19effc..a5bbeb6 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -69,7 +69,7 @@ RUN jupyter labextension update --all && \
     @bokeh/jupyter_bokeh \
     @jupyter-widgets/jupyterlab-manager \
     @pyviz/jupyterlab_pyviz \
-    @ryantam626/jupyterlab_code_formatter \
+    @ryantam626/jupyterlab_code_formatter@1.3.1 \
     dask-labextension \
     jupyterlab-plotly \
     plotlywidget && \
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 99f28f6..3b16e3b 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -12,7 +12,7 @@ dependencies:
   - ipyleaflet
   - ipympl
   - isort
-  - jupyterlab_code_formatter
+  - jupyterlab_code_formatter=1.3.1
   - nano
   - nb_conda_kernels
   - octave_kernel=0.32.0=pyh9f0ad1d_0

From 45e9174f0c11372d0fa51a09b7aff53ccbd62679 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 11:20:52 -0700
Subject: [PATCH 118/224] pin isort as well for jlab code formatter

---
 notebook/notebook_environment.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 3b16e3b..ee15ce9 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -11,7 +11,9 @@ dependencies:
   - ipdb
   - ipyleaflet
   - ipympl
-  - isort
+  # pinning isort and jupyterlab_code_formatter until there's a version of the latter
+  # on conda-forget that can support v5 of the former
+  - isort=4
   - jupyterlab_code_formatter=1.3.1
   - nano
   - nb_conda_kernels

From 9c0e59ae72657acc5f2f9cb733d0c022e60cf744 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 21:12:33 -0700
Subject: [PATCH 119/224] add sphinx_rtd_theme

---
 notebook/notebook_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index ee15ce9..0bc78b0 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -26,7 +26,7 @@ dependencies:
   - pytest-runner
   - python-graphviz
   - sidecar
-  - sphinx
+  - sphinx_rtd_theme
   - tox
   - pip:
     - black_nbconvert

From aef3b5a9361692148dad466e77d25106b0161471 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Thu, 9 Jul 2020 21:15:03 -0700
Subject: [PATCH 120/224] add ipypublish

---
 notebook/notebook_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 0bc78b0..60801a8 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -11,6 +11,7 @@ dependencies:
   - ipdb
   - ipyleaflet
   - ipympl
+  - ipypublish
   # pinning isort and jupyterlab_code_formatter until there's a version of the latter
   # on conda-forget that can support v5 of the former
   - isort=4

From c2cd5b5d83665eb806041c81e4f3290b1e8612b8 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 17:14:13 -0700
Subject: [PATCH 121/224] add nbdime and vim bindings

---
 notebook/Dockerfile               | 5 ++++-
 notebook/notebook_environment.yml | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index a5bbeb6..70df758 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -70,10 +70,13 @@ RUN jupyter labextension update --all && \
     @jupyter-widgets/jupyterlab-manager \
     @pyviz/jupyterlab_pyviz \
     @ryantam626/jupyterlab_code_formatter@1.3.1 \
+    @ryantam626/jupyterlab_vim \
     dask-labextension \
     jupyterlab-plotly \
+    nbdime-jupyterlab \
     plotlywidget && \
-    sudo jupyter serverextension enable --py jupyterlab_code_formatter --system
+    jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix && \
+    jupyter labextension disable jupyterlab_vim
 
 ##################
 ## clawpack added for coastal-specific image
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 60801a8..649bc5f 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -17,6 +17,7 @@ dependencies:
   - isort=4
   - jupyterlab_code_formatter=1.3.1
   - nano
+  - nbdime
   - nb_conda_kernels
   - octave_kernel=0.32.0=pyh9f0ad1d_0
   - oct2py=5.0.4=py_0

From 24d837a19fa9fa37648e7e7f7bba7b7a440519a0 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 17:39:56 -0700
Subject: [PATCH 122/224] separate out extension disable

---
 notebook/Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 70df758..bf15225 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -73,10 +73,10 @@ RUN jupyter labextension update --all && \
     @ryantam626/jupyterlab_vim \
     dask-labextension \
     jupyterlab-plotly \
-    nbdime-jupyterlab \
     plotlywidget && \
-    jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix && \
-    jupyter labextension disable jupyterlab_vim
+    jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
+
+RUN jupyter labextension disable jupyterlab_vim
 
 ##################
 ## clawpack added for coastal-specific image

From 7bc1672b3b80bf196c83ce3fa808012cb60ee2ef Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 18:16:44 -0700
Subject: [PATCH 123/224] try another disable

---
 notebook/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index bf15225..a8da659 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -76,7 +76,7 @@ RUN jupyter labextension update --all && \
     plotlywidget && \
     jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
 
-RUN jupyter labextension disable jupyterlab_vim
+RUN jupyter labextension disable @ryantam626/jupyterlab_vim
 
 ##################
 ## clawpack added for coastal-specific image

From c9ff0dcbebc47b0f4fc7492393fafe9153bb69af Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 18:35:36 -0700
Subject: [PATCH 124/224] install matio

---
 .travis.yml                           |  3 ++-
 notebook/Dockerfile                   | 12 ++++++-----
 octave-worker/Dockerfile              | 12 +++++++++++
 shared_resources/base_environment.yml |  1 +
 shared_resources/matio_73to5.c        | 31 +++++++++++++++++++++++++++
 worker/Dockerfile                     | 11 +++++-----
 6 files changed, 58 insertions(+), 12 deletions(-)
 create mode 100644 shared_resources/matio_73to5.c

diff --git a/.travis.yml b/.travis.yml
index b28f5eb..57a665a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,7 +19,8 @@ install:
       notebook/worker-template.yml; fi"
 - "rm notebook/worker-template.yml.bak"
 - "cat notebook/worker-template.yml | grep image:"
-- "cp shared_resources/* $IMAGE_NAME && chmod -R +x $IMAGE_NAME"
+- "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \
+    $IMAGE_NAME/shared_resources"
 - "cd $IMAGE_NAME"
 
 
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index a8da659..a9e985a 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -23,10 +23,7 @@ COPY add_service_creds.py /pre-home
 COPY run_sql_proxy.py /pre-home
 COPY config.yaml /pre-home
 
-RUN sudo mkdir /tempdir
-COPY common.sh /tempdir
-COPY Makefile.clawpack /tempdir
-COPY fix-permissions.sh /tempdir
+COPY shared_resources /tempdir
 
 COPY prepare.sh /usr/bin
 COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json
@@ -46,7 +43,7 @@ RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recomme
 
 # set up conda channels
 RUN mkdir /opt/conda/specs
-COPY base_environment.yml /opt/conda/specs
+COPY shared_resources/base_environment.yml /opt/conda/specs
 COPY notebook_environment.yml /opt/conda/specs
 RUN conda config --add channels conda-forge && \
   conda config --set channel_priority strict
@@ -62,6 +59,11 @@ RUN conda env update -f /opt/conda/specs/notebook_environment.yml
 
 RUN conda list -n base
 
+# Compile MatI/O
+RUN source /opt/conda/etc/profile.d/conda.sh \
+  && conda activate \
+  && g++ $CFLAGS -L/opt/conda/lib -lmatio /tempdir/matio_73to5.c -o \
+    /opt/conda/bin/matio_73to5
 
 ## Set up jupyter lab extensions
 RUN jupyter labextension update --all && \
diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile
index 5e16bde..0b72b44 100644
--- a/octave-worker/Dockerfile
+++ b/octave-worker/Dockerfile
@@ -2,6 +2,8 @@ ARG TRAVIS_COMMIT=${TRAVIS_COMMIT}
 FROM rhodium/worker:${TRAVIS_COMMIT}
 ARG DEBIAN_FRONTEND=noninteractive
 
+COPY shared_resources /tempdir
+
 ## filepath curation
 COPY octave_environment.yml /opt/conda/specs/octave_environment.yml
 
@@ -11,3 +13,13 @@ RUN apt-get update \
 
 # add octave-specific packages
 RUN conda env update -f /opt/conda/specs/octave_environment.yml
+
+# Compile MatI/O
+RUN source /opt/conda/etc/profile.d/conda.sh \
+  && conda activate \
+  && g++ $CFLAGS -L/opt/conda/lib -lmatio /tempdir/matio_73to5.c -o \
+    /opt/conda/bin/matio_73to5
+
+RUN rm -rf /tempdir \
+    && conda clean -yaf \
+    && sudo apt-get clean
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 143656a..58a391e 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -36,6 +36,7 @@ dependencies:
   - jedi=0.17.1=py37hc8dfbb8_0
   - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
   - kubernetes=1.18.5=0
+  - libmatio=1.5.12=hf54d351_1004
   - lz4=3.1.0=py37h5a7ed16_0
   - make=4.3=h516909a_0
   - matplotlib=3.2.2=0
diff --git a/shared_resources/matio_73to5.c b/shared_resources/matio_73to5.c
new file mode 100644
index 0000000..058a3b7
--- /dev/null
+++ b/shared_resources/matio_73to5.c
@@ -0,0 +1,31 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include "matio.h"
+
+int main(int argc, char **argv) {
+    mat_t    *matfp_in;
+    mat_t    *matfp_out;
+    matvar_t *matvar;
+    matfp_in = Mat_Open(argv[1], MAT_ACC_RDONLY);
+    if ( NULL == matfp_in ) {
+        fprintf(stderr, "Error opening MAT file \"%s\"!\n", argv[1]);
+        return EXIT_FAILURE;
+    }
+    matfp_out = Mat_CreateVer(argv[2], NULL, MAT_FT_MAT5);
+
+    while ( (matvar = Mat_VarReadNext(matfp_in)) != NULL ) {
+        printf("Writing MAT variable: %s... ", matvar->name);
+        if (0 != Mat_VarWrite(matfp_out, matvar, MAT_COMPRESSION_NONE)) {
+            printf("failed.\n");
+        } else {
+            printf("succeeded.\n");
+        }
+        Mat_VarFree(matvar);
+    }
+    
+    Mat_Close(matfp_out);
+    fprintf(stderr, "Wrote MAT file: \"%s\"\n", argv[2]);
+    
+    Mat_Close(matfp_in);
+    return EXIT_SUCCESS;
+}
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 37855d1..b403fb8 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -5,9 +5,7 @@ ENV NPY_DISTUTILS_APPEND_FLAGS=1
 ENV MPLBACKEND=Agg
 
 ## filepath curation
-RUN mkdir /tempdir
-COPY common.sh /tempdir
-COPY Makefile.clawpack /tempdir
+COPY shared_resources /tempdir
 
 COPY add_service_creds.py /usr/bin
 COPY prepare.sh /usr/bin
@@ -35,7 +33,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.2-Linux
 
 ## set up conda channels
 RUN mkdir /opt/conda/specs
-COPY base_environment.yml /opt/conda/specs
+COPY shared_resources/base_environment.yml /opt/conda/specs
 RUN conda config --add channels conda-forge && \
     conda config --set channel_priority strict
 
@@ -114,8 +112,9 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
 ##################
 
 ## clean up
-RUN rm -rf /var/lib/apt/lists/* /tempdir
-RUN conda clean -yaf
+RUN rm -rf /var/lib/apt/lists/* /tempdir \
+    && conda clean -yaf \
+    && sudo apt-get clean
 
 ## prepare container
 ENV OMP_NUM_THREADS=1

From 654876ffbefeeca3e3209df7969e019235e28d35 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 19:08:23 -0700
Subject: [PATCH 125/224] sudo disable extension

---
 notebook/Dockerfile | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index a9e985a..ac612be 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -68,17 +68,17 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
 ## Set up jupyter lab extensions
 RUN jupyter labextension update --all && \
     jupyter labextension install \
-    @bokeh/jupyter_bokeh \
-    @jupyter-widgets/jupyterlab-manager \
-    @pyviz/jupyterlab_pyviz \
-    @ryantam626/jupyterlab_code_formatter@1.3.1 \
-    @ryantam626/jupyterlab_vim \
-    dask-labextension \
-    jupyterlab-plotly \
-    plotlywidget && \
-    jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
-
-RUN jupyter labextension disable @ryantam626/jupyterlab_vim
+      @bokeh/jupyter_bokeh \
+      @jupyter-widgets/jupyterlab-manager \
+      @pyviz/jupyterlab_pyviz \
+      @ryantam626/jupyterlab_code_formatter@1.3.1 \
+      @ryantam626/jupyterlab_vim \
+      dask-labextension \
+      jupyterlab-plotly \
+      plotlywidget && \
+    jupyter serverextension enable --py jupyterlab_code_formatter nbdime \
+      --sys-prefix && \
+    sudo jupyter labextension disable @ryantam626/jupyterlab_vim
 
 ##################
 ## clawpack added for coastal-specific image

From 493cb1ed0b77f3217ed891e21b8e9ab55d8e5989 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 19:24:08 -0700
Subject: [PATCH 126/224] add papermill to nb

---
 notebook/notebook_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 649bc5f..c836c09 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -22,6 +22,7 @@ dependencies:
   - octave_kernel=0.32.0=pyh9f0ad1d_0
   - oct2py=5.0.4=py_0
   - openssh
+  - papermill
   - pip
   - pytest
   - pytest-cov

From 782feb91d0905d463c684c940d5665fd544ee8e7 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 19:45:17 -0700
Subject: [PATCH 127/224] black_nbconvert -> blackbook

---
 notebook/notebook_environment.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index c836c09..09594ca 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -3,6 +3,7 @@ channels:
   - conda-forge
 dependencies:
   - black
+  - blackbook
   - bump2version
   - coverage
   - dask-labextension
@@ -31,5 +32,3 @@ dependencies:
   - sidecar
   - sphinx_rtd_theme
   - tox
-  - pip:
-    - black_nbconvert

From 127d948d9dc4b0158de94aa076e0462649cd6ef8 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 20:02:48 -0700
Subject: [PATCH 128/224] skip vim bindings

---
 notebook/Dockerfile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index ac612be..2f08857 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -72,13 +72,10 @@ RUN jupyter labextension update --all && \
       @jupyter-widgets/jupyterlab-manager \
       @pyviz/jupyterlab_pyviz \
       @ryantam626/jupyterlab_code_formatter@1.3.1 \
-      @ryantam626/jupyterlab_vim \
       dask-labextension \
       jupyterlab-plotly \
       plotlywidget && \
-    jupyter serverextension enable --py jupyterlab_code_formatter nbdime \
-      --sys-prefix && \
-    sudo jupyter labextension disable @ryantam626/jupyterlab_vim
+    jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
 
 ##################
 ## clawpack added for coastal-specific image

From c534a1f1206236704867665731a6f113678db5dd Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 20:34:52 -0700
Subject: [PATCH 129/224] keep shared_resources back

---
 .travis.yml              | 2 --
 notebook/Dockerfile      | 2 +-
 octave-worker/Dockerfile | 2 +-
 worker/Dockerfile        | 2 +-
 4 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 57a665a..516bb45 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,8 +19,6 @@ install:
       notebook/worker-template.yml; fi"
 - "rm notebook/worker-template.yml.bak"
 - "cat notebook/worker-template.yml | grep image:"
-- "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \
-    $IMAGE_NAME/shared_resources"
 - "cd $IMAGE_NAME"
 
 
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 2f08857..7cc060e 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -23,7 +23,7 @@ COPY add_service_creds.py /pre-home
 COPY run_sql_proxy.py /pre-home
 COPY config.yaml /pre-home
 
-COPY shared_resources /tempdir
+COPY ../shared_resources /tempdir
 
 COPY prepare.sh /usr/bin
 COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json
diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile
index 0b72b44..cc93a48 100644
--- a/octave-worker/Dockerfile
+++ b/octave-worker/Dockerfile
@@ -2,7 +2,7 @@ ARG TRAVIS_COMMIT=${TRAVIS_COMMIT}
 FROM rhodium/worker:${TRAVIS_COMMIT}
 ARG DEBIAN_FRONTEND=noninteractive
 
-COPY shared_resources /tempdir
+COPY ../shared_resources /tempdir
 
 ## filepath curation
 COPY octave_environment.yml /opt/conda/specs/octave_environment.yml
diff --git a/worker/Dockerfile b/worker/Dockerfile
index b403fb8..90f95bf 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -5,7 +5,7 @@ ENV NPY_DISTUTILS_APPEND_FLAGS=1
 ENV MPLBACKEND=Agg
 
 ## filepath curation
-COPY shared_resources /tempdir
+COPY ../shared_resources /tempdir
 
 COPY add_service_creds.py /usr/bin
 COPY prepare.sh /usr/bin

From daef78119477e592b4fb245ff9eb571bd79853b5 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 20:42:09 -0700
Subject: [PATCH 130/224] undo last change

---
 .travis.yml              | 4 ++++
 notebook/Dockerfile      | 2 +-
 octave-worker/Dockerfile | 2 +-
 worker/Dockerfile        | 2 +-
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 516bb45..86bb3b1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,6 +19,10 @@ install:
       notebook/worker-template.yml; fi"
 - "rm notebook/worker-template.yml.bak"
 - "cat notebook/worker-template.yml | grep image:"
+- "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \
+    $IMAGE_NAME/shared_resources"
+- "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \
+    octave-worker/shared_resources && chmod -R +x octave-worker/shared_resources; fi"
 - "cd $IMAGE_NAME"
 
 
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 7cc060e..2f08857 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -23,7 +23,7 @@ COPY add_service_creds.py /pre-home
 COPY run_sql_proxy.py /pre-home
 COPY config.yaml /pre-home
 
-COPY ../shared_resources /tempdir
+COPY shared_resources /tempdir
 
 COPY prepare.sh /usr/bin
 COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json
diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile
index cc93a48..0b72b44 100644
--- a/octave-worker/Dockerfile
+++ b/octave-worker/Dockerfile
@@ -2,7 +2,7 @@ ARG TRAVIS_COMMIT=${TRAVIS_COMMIT}
 FROM rhodium/worker:${TRAVIS_COMMIT}
 ARG DEBIAN_FRONTEND=noninteractive
 
-COPY ../shared_resources /tempdir
+COPY shared_resources /tempdir
 
 ## filepath curation
 COPY octave_environment.yml /opt/conda/specs/octave_environment.yml
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 90f95bf..b403fb8 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -5,7 +5,7 @@ ENV NPY_DISTUTILS_APPEND_FLAGS=1
 ENV MPLBACKEND=Agg
 
 ## filepath curation
-COPY ../shared_resources /tempdir
+COPY shared_resources /tempdir
 
 COPY add_service_creds.py /usr/bin
 COPY prepare.sh /usr/bin

From 4118c59307077b00a15b14500544a1c81018f148 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 21:30:09 -0700
Subject: [PATCH 131/224] try again

---
 .travis.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 86bb3b1..2b06309 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,6 +28,9 @@ install:
 
 script:
 - docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
+- "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
+    rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \
+    ../octave-worker; fi"
 - docker run -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT bash /clawpack/run_tests.sh
 
 deploy:
@@ -65,10 +68,8 @@ deploy:
 # octave-worker builds
 - provider: script
   script: >-
-    docker build -t rhodium/octave-worker:$TRAVIS_COMMIT
-    --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT ../octave-worker &&
     docker tag rhodium/octave-worker:$TRAVIS_COMMIT
-    rhodium/octave-worker:$TRAVIS_BRANCH &&
+      rhodium/octave-worker:$TRAVIS_BRANCH &&
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/octave-worker:$TRAVIS_COMMIT" &&
     docker push "rhodium/octave-worker:$TRAVIS_BRANCH"
@@ -79,9 +80,9 @@ deploy:
 - provider: script
   script: >-
     docker tag rhodium/octave-worker:$TRAVIS_COMMIT
-    rhodium/octave-worker:dev &&
+      rhodium/octave-worker:dev &&
     docker tag rhodium/octave-worker:$TRAVIS_COMMIT
-    rhodium/octave-worker:latest &&
+      rhodium/octave-worker:latest &&
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/octave-worker:$TRAVIS_COMMIT" &&
     docker push "rhodium/octave-worker:dev" &&
@@ -93,7 +94,7 @@ deploy:
 - provider: script
   script: >-
     docker tag rhodium/octave-worker:$TRAVIS_COMMIT
-    rhodium/octave-worker:$TRAVIS_TAG &&
+      rhodium/octave-worker:$TRAVIS_TAG &&
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/octave-worker:$TRAVIS_TAG"
   on:

From 513efe003998b602f3c6d436e373e993f7aa5905 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 10 Jul 2020 22:01:33 -0700
Subject: [PATCH 132/224] trying some diff travis things

---
 .travis.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2b06309..94cdc30 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -69,7 +69,7 @@ deploy:
 - provider: script
   script: >-
     docker tag rhodium/octave-worker:$TRAVIS_COMMIT
-      rhodium/octave-worker:$TRAVIS_BRANCH &&
+    rhodium/octave-worker:$TRAVIS_BRANCH &&
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/octave-worker:$TRAVIS_COMMIT" &&
     docker push "rhodium/octave-worker:$TRAVIS_BRANCH"
@@ -80,9 +80,9 @@ deploy:
 - provider: script
   script: >-
     docker tag rhodium/octave-worker:$TRAVIS_COMMIT
-      rhodium/octave-worker:dev &&
+    rhodium/octave-worker:dev &&
     docker tag rhodium/octave-worker:$TRAVIS_COMMIT
-      rhodium/octave-worker:latest &&
+    rhodium/octave-worker:latest &&
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/octave-worker:$TRAVIS_COMMIT" &&
     docker push "rhodium/octave-worker:dev" &&
@@ -94,7 +94,7 @@ deploy:
 - provider: script
   script: >-
     docker tag rhodium/octave-worker:$TRAVIS_COMMIT
-      rhodium/octave-worker:$TRAVIS_TAG &&
+    rhodium/octave-worker:$TRAVIS_TAG &&
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
     docker push "rhodium/octave-worker:$TRAVIS_TAG"
   on:

From 142d6f86006da6f69899e4390ed1bf34bd16c584 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sat, 11 Jul 2020 10:10:14 -0700
Subject: [PATCH 133/224] pin rhg_compute_tools and pytides

---
 shared_resources/base_environment.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 58a391e..d1ac991 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -77,6 +77,6 @@ dependencies:
     - climate-toolbox==0.1.5
     - noaa-coops==0.1.5
     - parameterize-jobs==0.1.1
-    - rhg_compute_tools
-    - git+https://github.com/maritimeplanning/pytides.git@master
+    - rhg_compute_tools==0.2.2
+    - git+https://github.com/maritimeplanning/pytides.git@a8397a0e12645d22c62612385d18e63c4d67d6fa#egg=pytides
 name: base

From 4ba1167f924db05e46ef7ca7f5918606ceffa63c Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sat, 11 Jul 2020 17:12:23 -0700
Subject: [PATCH 134/224] add fuzzywuzzy

---
 shared_resources/base_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index d1ac991..5d58f99 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -22,6 +22,7 @@ dependencies:
   - fastparquet=0.4.0=py37h03ebfcd_0
   - fiona=1.8.13=py37h0492a4a_1
   - fusepy=3.0.1=py_0
+  - fuzzywuzzy=0.17.0=py_0
   - gcsfs=0.6.2=py_0
   - gdal=3.0.4=py37h4b180d9_10
   - geopandas=0.8.0=py_1

From 8675a5c2495e77d8d59b2993d4302e13fdafd889 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 12 Jul 2020 09:16:12 -0700
Subject: [PATCH 135/224] back to black_nbconvert

---
 notebook/notebook_environment.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 09594ca..bd53e04 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -3,7 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - black
-  - blackbook
   - bump2version
   - coverage
   - dask-labextension
@@ -32,3 +31,5 @@ dependencies:
   - sidecar
   - sphinx_rtd_theme
   - tox
+  - pip:
+    - black_nbconvert
\ No newline at end of file

From c865d2793c547c8b0364a0910322c826f77fc246 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 12 Jul 2020 21:09:01 -0700
Subject: [PATCH 136/224] make gitpython base package

---
 notebook/notebook_environment.yml     | 1 -
 shared_resources/base_environment.yml | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index bd53e04..a3aa1a6 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -7,7 +7,6 @@ dependencies:
   - coverage
   - dask-labextension
   - flake8
-  - gitpython
   - ipdb
   - ipyleaflet
   - ipympl
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 5d58f99..b347ca5 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -29,6 +29,7 @@ dependencies:
   - geotiff=1.6.0=h05acad5_0
   - geoviews=1.8.1=py_0
   - git=2.27.0=pl526h5e3e691_0
+  - gitpython=3.1.3=py_0
   - google-cloud-container=0.3.0=py37_0
   - google-cloud-storage=1.28.1=pyh9f0ad1d_0
   - holoviews=1.13.3=pyh9f0ad1d_0

From 2d959e48e9ab29d721a38c1945731594ddd06b07 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 12 Jul 2020 21:48:38 -0700
Subject: [PATCH 137/224] add pygeos

---
 shared_resources/base_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index b347ca5..834532b 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -52,6 +52,7 @@ dependencies:
   - pillow=7.2.0=py37h718be6c_0
   - pip=20.1.1=py_1
   - plotly=4.8.2=pyh9f0ad1d_0
+  - pygeos=0.7.1=py37hc88ce51_0
   - pyinterp=0.3.2=mkl_py37he448baa_102
   - pyshp=2.1.0=py_0
   - python=3.7.6=cpython_h8356626_6

From 7c650e7f622b7647e5d0ba2e79c46fd6198065cf Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 13 Jul 2020 06:57:53 -0700
Subject: [PATCH 138/224] install jupyterlab-flake8

---
 notebook/Dockerfile     | 1 +
 notebook/overrides.json | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 2f08857..edc6c75 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -73,6 +73,7 @@ RUN jupyter labextension update --all && \
       @pyviz/jupyterlab_pyviz \
       @ryantam626/jupyterlab_code_formatter@1.3.1 \
       dask-labextension \
+      jupyterlab-flake8 \
       jupyterlab-plotly \
       plotlywidget && \
     jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
diff --git a/notebook/overrides.json b/notebook/overrides.json
index 38e8912..10c1991 100755
--- a/notebook/overrides.json
+++ b/notebook/overrides.json
@@ -10,5 +10,8 @@
       "rulers": [88],
       "lineNumbers": true
     }
+  },
+  "jupyterlab-flake8:plugins": {
+    "toggled": false
   }
 }

From 2206913d133e0e9806bc3fc6974c5485622c83bf Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 13 Jul 2020 07:08:22 -0700
Subject: [PATCH 139/224] add flake8 settings and toggle on

---
 notebook/.flake8        | 2 ++
 notebook/overrides.json | 3 ---
 2 files changed, 2 insertions(+), 3 deletions(-)
 create mode 100644 notebook/.flake8

diff --git a/notebook/.flake8 b/notebook/.flake8
new file mode 100644
index 0000000..2bcd70e
--- /dev/null
+++ b/notebook/.flake8
@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 88
diff --git a/notebook/overrides.json b/notebook/overrides.json
index 10c1991..38e8912 100755
--- a/notebook/overrides.json
+++ b/notebook/overrides.json
@@ -10,8 +10,5 @@
       "rulers": [88],
       "lineNumbers": true
     }
-  },
-  "jupyterlab-flake8:plugins": {
-    "toggled": false
   }
 }

From f4bb8975e81da497d1d68638509540fb73aca392 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 13 Jul 2020 08:09:55 -0700
Subject: [PATCH 140/224] rm buggy jupyterlab-flake8

---
 notebook/.flake8    | 2 --
 notebook/Dockerfile | 1 -
 2 files changed, 3 deletions(-)
 delete mode 100644 notebook/.flake8

diff --git a/notebook/.flake8 b/notebook/.flake8
deleted file mode 100644
index 2bcd70e..0000000
--- a/notebook/.flake8
+++ /dev/null
@@ -1,2 +0,0 @@
-[flake8]
-max-line-length = 88
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index edc6c75..2f08857 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -73,7 +73,6 @@ RUN jupyter labextension update --all && \
       @pyviz/jupyterlab_pyviz \
       @ryantam626/jupyterlab_code_formatter@1.3.1 \
       dask-labextension \
-      jupyterlab-flake8 \
       jupyterlab-plotly \
       plotlywidget && \
     jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix

From 1e5bac968da06814ab2533044ea7ae9aab1e42c2 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 13 Jul 2020 08:36:24 -0700
Subject: [PATCH 141/224] add variable insepctor

---
 notebook/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 2f08857..fcac195 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -70,6 +70,7 @@ RUN jupyter labextension update --all && \
     jupyter labextension install \
       @bokeh/jupyter_bokeh \
       @jupyter-widgets/jupyterlab-manager \
+      @lckr/jupyterlab_variableinspector \
       @pyviz/jupyterlab_pyviz \
       @ryantam626/jupyterlab_code_formatter@1.3.1 \
       dask-labextension \

From b337cc9260da0b4e84e4646506842e864cdee949 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 14 Jul 2020 08:59:15 -0700
Subject: [PATCH 142/224] bump xarray to 0.16

---
 shared_resources/base_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 834532b..b119344 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -70,7 +70,7 @@ dependencies:
   - sparse=0.10.0=py_0
   - statsmodels=0.11.1=py37h8f50634_2
   - tini=0.18.0=h14c3975_1001
-  - xarray=0.14.1=py_1                 # pinkeep: xarray=0.14
+  - xarray=0.16.0=py_0
   - xesmf=0.3.0=py_0
   - xhistogram=0.1.1=py_0
   - xlrd=1.2.0=py_0

From 6da704a94e5474b07d432fad9953135d4b8e78ef Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 14 Jul 2020 09:05:12 -0700
Subject: [PATCH 143/224] bump several packages

---
 notebook/Dockerfile                   |  2 +-
 notebook/notebook_environment.yml     |  6 ++----
 shared_resources/base_environment.yml | 14 +++++++-------
 3 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index fcac195..b06f8b2 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -72,7 +72,7 @@ RUN jupyter labextension update --all && \
       @jupyter-widgets/jupyterlab-manager \
       @lckr/jupyterlab_variableinspector \
       @pyviz/jupyterlab_pyviz \
-      @ryantam626/jupyterlab_code_formatter@1.3.1 \
+      @ryantam626/jupyterlab_code_formatter \
       dask-labextension \
       jupyterlab-plotly \
       plotlywidget && \
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index a3aa1a6..f8ee86e 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -11,10 +11,8 @@ dependencies:
   - ipyleaflet
   - ipympl
   - ipypublish
-  # pinning isort and jupyterlab_code_formatter until there's a version of the latter
-  # on conda-forget that can support v5 of the former
-  - isort=4
-  - jupyterlab_code_formatter=1.3.1
+  - isort
+  - jupyterlab_code_formatter
   - nano
   - nbdime
   - nb_conda_kernels
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index b119344..ff6f55b 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -2,15 +2,15 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4=4.9.1=py37hc8dfbb8_0
-  - blas=2.14=openblas
+  - blas=2.17=openblas
   - bottleneck=1.3.2=py37h03ebfcd_1
-  - boto3=1.14.16=pyh9f0ad1d_0
-  - bqplot=0.12.13=pyh9f0ad1d_0
+  - boto3=1.14.20=pyh9f0ad1d_0
+  - bqplot=0.12.14=pyh9f0ad1d_0
   - cachecontrol=0.12.6=py_0
   - cartopy=0.18.0=py37h4b180d9_0
   - cftime=1.1.3=py37h03ebfcd_0
   - click=7.1.2=pyh9f0ad1d_0
-  - compilers=1.0.4=0
+  - compilers=1.1.0=0
   - dask=2.20.0=py_0
   - dask-glm=0.2.0=py_1
   - dask-kubernetes=0.10.1=py_0
@@ -34,14 +34,14 @@ dependencies:
   - google-cloud-storage=1.28.1=pyh9f0ad1d_0
   - holoviews=1.13.3=pyh9f0ad1d_0
   - h5netcdf=0.8.0=py_0
-  - intake-esm=2020.3.16.2=py_0
+  - intake-esm=2020.6.11=py_0
   - jedi=0.17.1=py37hc8dfbb8_0
   - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
   - kubernetes=1.18.5=0
   - libmatio=1.5.12=hf54d351_1004
   - lz4=3.1.0=py37h5a7ed16_0
   - make=4.3=h516909a_0
-  - matplotlib=3.2.2=0
+  - matplotlib=3.2.2=1
   - nc-time-axis=1.2.0=py_1
   - netcdf-fortran=4.5.3=nompi_hfef6a68_100
   - netcdf4=1.5.3=nompi_py37hdc49583_105
@@ -49,7 +49,7 @@ dependencies:
   - numba=0.49.1=py37h0da4684_0
   - pandas=1.0.5=py37h0da4684_0
   - phantomjs=2.1.1=1 # for geoviews
-  - pillow=7.2.0=py37h718be6c_0
+  - pillow=7.2.0=py37h718be6c_1
   - pip=20.1.1=py_1
   - plotly=4.8.2=pyh9f0ad1d_0
   - pygeos=0.7.1=py37hc88ce51_0

From d55edd0aaaa5c11e8084f7094723cab92ff0a33b Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 14 Jul 2020 11:54:57 -0700
Subject: [PATCH 144/224] pin jupyterhub

---
 notebook/notebook_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index f8ee86e..ffaea12 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -12,6 +12,7 @@ dependencies:
   - ipympl
   - ipypublish
   - isort
+  - jupyterhub=1.0.0  # pinkeep: jupyterhub=1.0.0
   - jupyterlab_code_formatter
   - nano
   - nbdime

From 86ed8624fe65e8515e16c63fdcfa63b040a86ba1 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Wed, 15 Jul 2020 06:54:11 -0700
Subject: [PATCH 145/224] update base images

---
 notebook/Dockerfile               | 2 +-
 notebook/notebook_environment.yml | 1 +
 worker/Dockerfile                 | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index b06f8b2..ab5e749 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook@sha256:135779f39abfdfe75a6dbc50c7a15c55cc79c8231cd75f83aaabf87127c60adf
+FROM jupyter/base-notebook@sha256:58cfa094f8263b37d24ec357d0b9ca6227402e6ccc1229f097b843de4487d3a2
 ARG DEBIAN_FRONTEND=noninteractive
 
 ## needed to make sure things with cython compile correctly
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index ffaea12..0ceed04 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -12,6 +12,7 @@ dependencies:
   - ipympl
   - ipypublish
   - isort
+# need jupyterhub version to match that running on the hub
   - jupyterhub=1.0.0  # pinkeep: jupyterhub=1.0.0
   - jupyterlab_code_formatter
   - nano
diff --git a/worker/Dockerfile b/worker/Dockerfile
index b403fb8..8fd9faa 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -1,5 +1,5 @@
-## using same container as jupyter/base-notebook:python-3.7.3
-FROM ubuntu:focal-20200423@sha256:238e696992ba9913d24cfc3727034985abd136e08ee3067982401acdc30cbf3f
+## using same base image as the "base image for the base image" of the notebook image
+FROM ubuntu:focal-20200606@sha256:93fd0705706e5bdda6cc450b384d8d5afb18fecc19e054fe3d7a2c8c2aeb2c83
 ARG DEBIAN_FRONTEND=noninteractive
 ENV NPY_DISTUTILS_APPEND_FLAGS=1
 ENV MPLBACKEND=Agg

From 3be4b665f9e38d4d67d829d93ade40686a40e853 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 17 Jul 2020 19:52:24 -0700
Subject: [PATCH 146/224] include descartes for gpd plotting

---
 shared_resources/base_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index ff6f55b..71fe544 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -16,6 +16,7 @@ dependencies:
   - dask-kubernetes=0.10.1=py_0
   - dask-ml=1.5.0=py_0
   - datashader=0.10.0=py_0
+  - descartes=1.1.0=py_4
   - distributed=2.20.0=py37hc8dfbb8_0
   - dropbox=10.1.1=pyh9f0ad1d_0
   - esmpy=8.0.1=nompi_py37h777d1d2_0

From 0fbb0a98cbe518e0e9f0767bbc7bdc9bbfe501c7 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Wed, 22 Jul 2020 07:47:10 -0700
Subject: [PATCH 147/224] add hvplot

---
 shared_resources/base_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 71fe544..4ef3295 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -34,6 +34,7 @@ dependencies:
   - google-cloud-container=0.3.0=py37_0
   - google-cloud-storage=1.28.1=pyh9f0ad1d_0
   - holoviews=1.13.3=pyh9f0ad1d_0
+  - hvplot=0.6.0=pyh9f0ad1d_0
   - h5netcdf=0.8.0=py_0
   - intake-esm=2020.6.11=py_0
   - jedi=0.17.1=py37hc8dfbb8_0

From f79ae4dc4dc438dc283ab57a69f30c94b469140e Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Wed, 22 Jul 2020 10:10:36 -0700
Subject: [PATCH 148/224] add JRE

---
 shared_resources/common.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shared_resources/common.sh b/shared_resources/common.sh
index 6b97bcc..94d208e 100755
--- a/shared_resources/common.sh
+++ b/shared_resources/common.sh
@@ -8,6 +8,7 @@ apt-get install -yq --no-install-recommends \
   unzip \
   ca-certificates \
   curl \
+  default-jre \
   lsb-release \
   gnupg2 \
   sudo \

From 745fea6948b7e25e0dcd614ffa6d81ceb9ac661e Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sat, 25 Jul 2020 16:22:55 -0700
Subject: [PATCH 149/224] add toc to notebook

---
 notebook/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index ab5e749..07cc0f6 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -69,6 +69,7 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
 RUN jupyter labextension update --all && \
     jupyter labextension install \
       @bokeh/jupyter_bokeh \
+      @jupyterlab/toc \
       @jupyter-widgets/jupyterlab-manager \
       @lckr/jupyterlab_variableinspector \
       @pyviz/jupyterlab_pyviz \

From f7e3ff466d5041b5a8b058ceccd0929902205c27 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 31 Jul 2020 12:46:09 -0700
Subject: [PATCH 150/224] fix pin.py for new base env loc

---
 pin.py | 39 +++++++++++++++++----------------------
 1 file changed, 17 insertions(+), 22 deletions(-)

diff --git a/pin.py b/pin.py
index 175ab22..b2b426e 100644
--- a/pin.py
+++ b/pin.py
@@ -25,6 +25,13 @@
 from ruamel.yaml import YAML
 
 
+SPEC_FILES = [
+            ('shared_resources/base_environment.yml', 'base'),
+            ('notebook/notebook_environment.yml', 'base'),
+            ('octave-worker/octave_environment.yml', 'base'),
+            ('notebook/r_environment.yml', 'r')]
+
+
 def get_versions_in_current_environment(envname='base'):
     '''
     Calls ``conda env export -n {envname} --json`` and returns spec
@@ -294,22 +301,16 @@ def pinversions():
 def pin(file, dry_run):
     '''Pin packages in environment files based on environments on the local machine'''
     
-    spec_files = [
-            ('base_environment.yml', 'base'),
-            ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base'),
-            ('notebook/r_environment.yml', 'r')]
-    
     if file == 'all':
-        pin_files(spec_files, dry_run=dry_run)
+        pin_files(SPEC_FILES, dry_run=dry_run)
     elif file == 'base':
-        pin_files([spec_files[0]], dry_run=dry_run)
+        pin_files([SPEC_FILES[0]], dry_run=dry_run)
     elif file == 'notebook':
-        pin_files([spec_files[1]], dry_run=dry_run)
+        pin_files([SPEC_FILES[1]], dry_run=dry_run)
     elif file == 'octave':
-        pin_files([spec_files[2]], dry_run=dry_run)
+        pin_files([SPEC_FILES[2]], dry_run=dry_run)
     elif file == 'r':
-        pin_files([spec_files[3]], dry_run=dry_run)
+        pin_files([SPEC_FILES[3]], dry_run=dry_run)
     else:
         raise ValueError(
             'env type not recognized: {}'
@@ -329,22 +330,16 @@ def pin(file, dry_run):
 def unpin(file, dry_run):
     '''Unpin packages in environment files'''
     
-    spec_files = [
-            ('base_environment.yml', 'base'),
-            ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base'),
-            ('notebook/r_environment.yml', 'r')]
-    
     if file == 'all':
-        unpin_files(spec_files, dry_run=dry_run)
+        unpin_files(SPEC_FILES, dry_run=dry_run)
     elif file == 'base':
-        unpin_files([spec_files[0]], dry_run=dry_run)
+        unpin_files([SPEC_FILES[0]], dry_run=dry_run)
     elif file == 'notebook':
-        unpin_files([spec_files[1]], dry_run=dry_run)
+        unpin_files([SPEC_FILES[1]], dry_run=dry_run)
     elif file == 'octave':
-        unpin_files([spec_files[2]], dry_run=dry_run)
+        unpin_files([SPEC_FILES[2]], dry_run=dry_run)
     elif file == 'r':
-        unpin_files([spec_files[3]], dry_run=dry_run)
+        unpin_files([SPEC_FILES[3]], dry_run=dry_run)
     else:
         raise ValueError(
             'env type not recognized: {}'

From b01e90648cc702cedc467b395265309508114041 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 31 Jul 2020 12:46:45 -0700
Subject: [PATCH 151/224] update to new jupyter image

---
 notebook/Dockerfile |  2 +-
 worker/Dockerfile   | 49 ++++++++++++++++++++++++++++-----------------
 2 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 07cc0f6..6c8d777 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook@sha256:58cfa094f8263b37d24ec357d0b9ca6227402e6ccc1229f097b843de4487d3a2
+FROM jupyter/base-notebook@sha256:b5abe43c6d312b5a7dd63e33f2291df52cddfc1ac15547542ac8b0990a30b0e5
 ARG DEBIAN_FRONTEND=noninteractive
 
 ## needed to make sure things with cython compile correctly
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 8fd9faa..0c7addd 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -1,28 +1,48 @@
 ## using same base image as the "base image for the base image" of the notebook image
-FROM ubuntu:focal-20200606@sha256:93fd0705706e5bdda6cc450b384d8d5afb18fecc19e054fe3d7a2c8c2aeb2c83
+FROM ubuntu:focal-20200703@sha256:d5a6519d9f048100123c568eb83f7ef5bfcad69b01424f420f17c932b00dea76
 ARG DEBIAN_FRONTEND=noninteractive
+
+# needed to properly install packages that use numpy libraries
 ENV NPY_DISTUTILS_APPEND_FLAGS=1
+
+# needed so that matplotlib will work headless
 ENV MPLBACKEND=Agg
 
+# from jupyter/base-notebook
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV CONDA_DIR=/opt/conda \
+    SHELL=/bin/bash \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
+ENV PATH=$CONDA_DIR/bin:$PATH
+
 ## filepath curation
 COPY shared_resources /tempdir
-
 COPY add_service_creds.py /usr/bin
 COPY prepare.sh /usr/bin
 
-
 ## perform a bunch of common actions
 RUN bash /tempdir/common.sh
 
 
 ###########
-## install miniconda
-# adapted from continuumio/miniconda3
-
-ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
-ENV PATH /opt/conda/bin:$PATH
-
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.2-Linux-x86_64.sh -O ~/miniconda.sh && \
+## install miniconda (following jupyter/base-notebook
+ENV MINICONDA_VERSION=4.8.3 \
+    MINICONDA_MD5=d63adf39f2c220950a063e0529d4ff74 \
+    CONDA_VERSION=4.8.3
+
+WORKDIR /tmp
+RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    echo "${MINICONDA_MD5} *Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
+    /bin/bash Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    conda config --system --prepend channels conda-forge && \
+    conda config --system --set channel_priority strict && \
+    conda update -n base --yes conda \
+    conda update --all --yes &&
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.8.3-Linux-x86_64.sh -O ~/miniconda.sh && \
     /bin/bash ~/miniconda.sh -b -p /opt/conda && \
     rm ~/miniconda.sh && \
     ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
@@ -30,16 +50,9 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.2-Linux
     echo "conda activate base" >> ~/.bashrc
 ###########
 
-
-## set up conda channels
+## set up python env
 RUN mkdir /opt/conda/specs
 COPY shared_resources/base_environment.yml /opt/conda/specs
-RUN conda config --add channels conda-forge && \
-    conda config --set channel_priority strict
-
-
-## set up python env
-RUN conda update -n base conda
 RUN conda env update -f /opt/conda/specs/base_environment.yml
 RUN conda list -n base
 

From 1c6282e0616c927bacdf3fdb1633eed7a85fcbd0 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 31 Jul 2020 14:10:24 -0700
Subject: [PATCH 152/224] bump to py38

---
 notebook/notebook_environment.yml     | 52 +++++++--------
 shared_resources/base_environment.yml | 94 +++++++++++++--------------
 2 files changed, 73 insertions(+), 73 deletions(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 0ceed04..31f9a3a 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -2,33 +2,33 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - black
-  - bump2version
-  - coverage
-  - dask-labextension
-  - flake8
-  - ipdb
-  - ipyleaflet
-  - ipympl
-  - ipypublish
-  - isort
+  - black=19.10b0=py_4
+  - bump2version=1.0.0=pyh9f0ad1d_0
+  - coverage=5.2.1=py38h1e0a361_0
+  - dask-labextension=2.0.2=py_0
+  - flake8=3.8.3=py_1
+  - ipdb=0.13.3=pyh9f0ad1d_0
+  - ipyleaflet=0.13.3=pyh9f0ad1d_0
+  - ipympl=0.5.7=pyh9f0ad1d_1
+  - ipypublish=0.10.11=py38h32f6830_0
+  - isort=5.2.2=py38h32f6830_0
 # need jupyterhub version to match that running on the hub
-  - jupyterhub=1.0.0  # pinkeep: jupyterhub=1.0.0
-  - jupyterlab_code_formatter
-  - nano
-  - nbdime
-  - nb_conda_kernels
+  - jupyterhub=1.0.0=py38_0 # pinkeep: jupyterhub=1.0.0
+  - jupyterlab_code_formatter=1.3.5=py_0
+  - nano=2.9.8=hddfc1eb_1001
+  - nbdime=2.0.0=py_1
+  - nb_conda_kernels=2.2.3=py38_0
   - octave_kernel=0.32.0=pyh9f0ad1d_0
   - oct2py=5.0.4=py_0
-  - openssh
-  - papermill
-  - pip
-  - pytest
-  - pytest-cov
-  - pytest-runner
-  - python-graphviz
-  - sidecar
-  - sphinx_rtd_theme
-  - tox
+  - openssh=8.3p1=h5957347_0
+  - papermill=2.1.2=py38h32f6830_0
+  - pip=20.2=py_0
+  - pytest=6.0.1=py38h32f6830_0
+  - pytest-cov=2.10.0=pyh9f0ad1d_0
+  - pytest-runner=5.2=py_0
+  - python-graphviz=0.14.1=pyh9f0ad1d_0
+  - sidecar=0.4.0=pyh9f0ad1d_0
+  - sphinx_rtd_theme=0.5.0=pyh9f0ad1d_0
+  - tox=3.18.1=py38h32f6830_0
   - pip:
-    - black_nbconvert
\ No newline at end of file
+    - black_nbconvert
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 4ef3295..71fc2b8 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -1,85 +1,85 @@
 channels:
   - conda-forge
 dependencies:
-  - beautifulsoup4=4.9.1=py37hc8dfbb8_0
-  - blas=2.17=openblas
-  - bottleneck=1.3.2=py37h03ebfcd_1
-  - boto3=1.14.20=pyh9f0ad1d_0
+  - beautifulsoup4=4.9.1=py38h32f6830_0
+  - blas=2.14=openblas
+  - bottleneck=1.3.2=py38h8790de6_1
+  - boto3=1.14.32=pyh9f0ad1d_0
   - bqplot=0.12.14=pyh9f0ad1d_0
   - cachecontrol=0.12.6=py_0
-  - cartopy=0.18.0=py37h4b180d9_0
-  - cftime=1.1.3=py37h03ebfcd_0
+  - cartopy=0.18.0=py38h172510d_0
+  - cftime=1.2.1=py38h8790de6_0
   - click=7.1.2=pyh9f0ad1d_0
-  - compilers=1.1.0=0
-  - dask=2.20.0=py_0
+  - compilers=1.1.1=0
+  - dask=2.21.0=py_0
   - dask-glm=0.2.0=py_1
   - dask-kubernetes=0.10.1=py_0
-  - dask-ml=1.5.0=py_0
+  - dask-ml=1.6.0=py_0
   - datashader=0.10.0=py_0
   - descartes=1.1.0=py_4
-  - distributed=2.20.0=py37hc8dfbb8_0
+  - distributed=2.21.0=py38h32f6830_0
   - dropbox=10.1.1=pyh9f0ad1d_0
-  - esmpy=8.0.1=nompi_py37h777d1d2_0
-  - fastparquet=0.4.0=py37h03ebfcd_0
-  - fiona=1.8.13=py37h0492a4a_1
+  - esmpy=8.0.1=nompi_py38hbd9704b_0
+  - fastparquet=0.4.1=py38h8790de6_0
+  - fiona=1.8.13=py38h033e0f6_1
   - fusepy=3.0.1=py_0
   - fuzzywuzzy=0.17.0=py_0
   - gcsfs=0.6.2=py_0
-  - gdal=3.0.4=py37h4b180d9_10
-  - geopandas=0.8.0=py_1
+  - gdal=3.0.4=py38h172510d_10
+  - geopandas=0.8.1=py_0
   - geotiff=1.6.0=h05acad5_0
   - geoviews=1.8.1=py_0
-  - git=2.27.0=pl526h5e3e691_0
+  - git=2.28.0=pl526h5e3e691_0
   - gitpython=3.1.3=py_0
-  - google-cloud-container=0.3.0=py37_0
-  - google-cloud-storage=1.28.1=pyh9f0ad1d_0
+  - google-cloud-container=0.3.0=py38_0
+  - google-cloud-storage=1.30.0=pyh9f0ad1d_0
   - holoviews=1.13.3=pyh9f0ad1d_0
   - hvplot=0.6.0=pyh9f0ad1d_0
-  - h5netcdf=0.8.0=py_0
+  - h5netcdf=0.8.1=py_0
   - intake-esm=2020.6.11=py_0
-  - jedi=0.17.1=py37hc8dfbb8_0
+  - jedi=0.17.2=py38h32f6830_0
   - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
-  - kubernetes=1.18.5=0
+  - kubernetes=1.18.6=0
   - libmatio=1.5.12=hf54d351_1004
-  - lz4=3.1.0=py37h5a7ed16_0
+  - lz4=3.1.0=py38h66f7c9e_0
   - make=4.3=h516909a_0
-  - matplotlib=3.2.2=1
+  - matplotlib=3.3.0=1
   - nc-time-axis=1.2.0=py_1
   - netcdf-fortran=4.5.3=nompi_hfef6a68_100
-  - netcdf4=1.5.3=nompi_py37hdc49583_105
-  - nose=1.3.7=py37hc8dfbb8_1004
-  - numba=0.49.1=py37h0da4684_0
-  - pandas=1.0.5=py37h0da4684_0
+  - netcdf4=1.5.4=nompi_py38hfd55d45_100
+  - nose=1.3.7=py38h32f6830_1004
+  - numba=0.50.1=py38hcb8c335_1
+  - pandas=1.1.0=py38h950e882_0
   - phantomjs=2.1.1=1 # for geoviews
-  - pillow=7.2.0=py37h718be6c_1
-  - pip=20.1.1=py_1
-  - plotly=4.8.2=pyh9f0ad1d_0
-  - pygeos=0.7.1=py37hc88ce51_0
-  - pyinterp=0.3.2=mkl_py37he448baa_102
+  - pillow=7.2.0=py38h9776b28_1
+  - pip=20.2=py_0
+  - plotly=4.9.0=pyh9f0ad1d_0
+  - pygeos=0.7.1=py38hd168ffb_0
+  - pyinterp=0.3.2=mkl_py38hefd7b46_102
   - pyshp=2.1.0=py_0
-  - python=3.7.6=cpython_h8356626_6
-  - python-snappy=0.5.4=py37h7cfaab3_1
-  - pyyaml=5.3.1=py37h8f50634_0
-  - rasterio=1.1.5=py37h0492a4a_0
-  - regionmask=0.5.0=py_1
-  - rtree=0.9.4=py37h8526d28_1
-  - scikit-image=0.17.2=py37h0da4684_1
-  - scikit-learn=0.23.1=py37h8a51577_0
-  - scipy=1.5.0=py37ha3d9a3c_0
+  - python=3.8.5=h4d41432_2_cpython
+  - python-snappy=0.5.4=py38h7cfaab3_1
+  - pyyaml=5.3.1=py38h1e0a361_0
+  - rasterio=1.1.5=py38h033e0f6_1
+  - regionmask=0.6.0=py_0
+  - rtree=0.9.4=py38h08f867b_1
+  - scikit-image=0.17.2=py38hcb8c335_1
+  - scikit-learn=0.23.1=py38h3a94b23_0
+  - scipy=1.5.2=py38h8c5af15_0
   - seaborn=0.10.1=1
-  - selenium=3.141.0=py37h8f50634_1001 # for geoviews
-  - shapely=1.7.0=py37hc88ce51_3
+  - selenium=3.141.0=py38h1e0a361_1001 # for geoviews
+  - shapely=1.7.0=py38hd168ffb_3
   - sparse=0.10.0=py_0
-  - statsmodels=0.11.1=py37h8f50634_2
+  - statsmodels=0.11.1=py38h1e0a361_2
   - tini=0.18.0=h14c3975_1001
   - xarray=0.16.0=py_0
   - xesmf=0.3.0=py_0
   - xhistogram=0.1.1=py_0
-  - xlrd=1.2.0=py_0
+  - xlrd=1.2.0=pyh9f0ad1d_1
   - zarr=2.4.0=py_0
-  - zeromq=4.3.2=he1b5a44_2
+  - zeromq=4.3.2=he1b5a44_3
   - pip:
-    - climate-toolbox==0.1.5
+    - climate-toolbox==0.1.5 
     - noaa-coops==0.1.5
     - parameterize-jobs==0.1.1
     - rhg_compute_tools==0.2.2

From 68886232d36e6f0e6b655c441c47ee50e4b2ef11 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 31 Jul 2020 14:58:34 -0700
Subject: [PATCH 153/224] add locales to match jupyter base-notebook

---
 notebook/Dockerfile        | 39 ++-------------------
 shared_resources/common.sh |  1 +
 worker/Dockerfile          | 69 ++++++++------------------------------
 3 files changed, 17 insertions(+), 92 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 6c8d777..e62337d 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -88,44 +88,9 @@ ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
 ENV LIB_PATHS=/opt/conda/lib
-# Was getting weird behavior with any broad optimization flags (-O, -O2, -O3) wherein
-# the boundary velocity error might be triggered on different boundaries if you piped
-# both stdout and stderr to a file. If you only piped one or didn't pipe any, you 
-# got one answer and if you piped both you got a different answer. Without the flags,
-# you would get the same answer regardless (the one originally obtained when piping
-# both out and err). To maintain a level of optimization, I just took all of the flags
-# from O3 here: https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html and put them
-# in individually. A few were removed due to compile or runtime errors. In addition, I
-# added some flags applied by default from the conda gfortran compiler (-ftree-vectorize 
-# -fPIC -fno-plt -ffunction-sections). I also added the march flag for the google 
-# machines we are using (broadwell currently), and dropped fstack-protector-strong from
-# the list of conda-based flags, as it was throwing segfaults.
 ENV CLAW_FFLAGS="-DNETCDF -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include \
-  -fauto-inc-dec -fbranch-count-reg -fcombine-stack-adjustments -fcompare-elim \
-  -fcprop-registers -fdce -fdefer-pop -fdse -fforward-propagate \
-  -fguess-branch-probability -fif-conversion -fif-conversion2 \
-  -finline-functions-called-once -fipa-profile -fipa-pure-const -fipa-reference \
-  -fmerge-constants -fmove-loop-invariants -fomit-frame-pointer -freorder-blocks \
-  -fshrink-wrap -fshrink-wrap-separate -fsplit-wide-types -fssa-backprop -fssa-phiopt \
-  -ftree-bit-ccp -ftree-ccp -ftree-ch -ftree-coalesce-vars -ftree-copy-prop -ftree-dce \
-  -ftree-dominator-opts -ftree-dse -ftree-forwprop -ftree-fre -ftree-phiprop \
-  -ftree-pta -ftree-scev-cprop -ftree-sink -ftree-slsr -ftree-sra -ftree-ter \
-  -funit-at-a-time -freorder-blocks-algorithm=stc -freorder-blocks-and-partition \
-  -freorder-functions -frerun-cse-after-loop -fschedule-insns -fschedule-insns2 \
-  -fsched-interblock -fsched-spec -fstore-merging -fstrict-aliasing -fthread-jumps \
-  -ftree-builtin-call-dce -ftree-pre -ftree-switch-conversion  -ftree-tail-merge \
-  -ftree-vrp -fgcse  -fgcse-lm  -fhoist-adjacent-loads -finline-functions \
-  -finline-small-functions -findirect-inlining -fipa-bit-cp  -fipa-cp  -fipa-icf \
-  -fipa-ra  -fipa-sra  -fipa-vrp -fisolate-erroneous-paths-dereference -flra-remat \
-  -foptimize-sibling-calls -foptimize-strlen -fpartial-inlining -fpeephole2 \
-  -falign-functions -falign-jumps -falign-labels  -falign-loops -fcaller-saves \
-  -fcode-hoisting -fcrossjumping -fcse-follow-jumps  -fcse-skip-blocks \
-  -fdelete-null-pointer-checks -fdevirtualize  -fdevirtualize-speculatively \
-  -fexpensive-optimizations -fgcse-after-reload -floop-unroll-and-jam -fpeel-loops \
-  -fpredictive-commoning -fsplit-loops -fsplit-paths -ftree-loop-distribution \
-  -ftree-loop-vectorize -ftree-partial-pre -ftree-slp-vectorize -funswitch-loops \
-  -fvect-cost-model -fvect-cost-model=dynamic -march=broadwell -ftree-vectorize -fPIC \
-  -fno-plt -ffunction-sections"
+  -O3 -march=broadwell -ftree-vectorize -fPIC -fno-plt -ffunction-sections \
+  -fstack-protector-strong"
 
 # need to change shell in order for source command to work
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
diff --git a/shared_resources/common.sh b/shared_resources/common.sh
index 94d208e..69f784f 100755
--- a/shared_resources/common.sh
+++ b/shared_resources/common.sh
@@ -9,6 +9,7 @@ apt-get install -yq --no-install-recommends \
   ca-certificates \
   curl \
   default-jre \
+  locales \
   lsb-release \
   gnupg2 \
   sudo \
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 0c7addd..c3f607c 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -8,15 +8,6 @@ ENV NPY_DISTUTILS_APPEND_FLAGS=1
 # needed so that matplotlib will work headless
 ENV MPLBACKEND=Agg
 
-# from jupyter/base-notebook
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-ENV CONDA_DIR=/opt/conda \
-    SHELL=/bin/bash \
-    LC_ALL=en_US.UTF-8 \
-    LANG=en_US.UTF-8 \
-    LANGUAGE=en_US.UTF-8
-ENV PATH=$CONDA_DIR/bin:$PATH
-
 ## filepath curation
 COPY shared_resources /tempdir
 COPY add_service_creds.py /usr/bin
@@ -25,9 +16,19 @@ COPY prepare.sh /usr/bin
 ## perform a bunch of common actions
 RUN bash /tempdir/common.sh
 
-
 ###########
 ## install miniconda (following jupyter/base-notebook
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    locale-gen
+    
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV CONDA_DIR=/opt/conda \
+    SHELL=/bin/bash \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
+ENV PATH=$CONDA_DIR/bin:$PATH
+
 ENV MINICONDA_VERSION=4.8.3 \
     MINICONDA_MD5=d63adf39f2c220950a063e0529d4ff74 \
     CONDA_VERSION=4.8.3
@@ -40,14 +41,7 @@ RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py38_${MINICONDA
     conda config --system --prepend channels conda-forge && \
     conda config --system --set channel_priority strict && \
     conda update -n base --yes conda \
-    conda update --all --yes &&
-
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.8.3-Linux-x86_64.sh -O ~/miniconda.sh && \
-    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
-    rm ~/miniconda.sh && \
-    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-    echo "conda activate base" >> ~/.bashrc
+    conda update --all --yes
 ###########
 
 ## set up python env
@@ -65,44 +59,9 @@ ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran
 ENV LIB_PATHS=/opt/conda/lib
-# Was getting weird behavior with any broad optimization flags (-O, -O2, -O3) wherein
-# the boundary velocity error might be triggered on different boundaries if you piped
-# both stdout and stderr to a file. If you only piped one or didn't pipe any, you 
-# got one answer and if you piped both you got a different answer. Without the flags,
-# you would get the same answer regardless (the one originally obtained when piping
-# both out and err). To maintain a level of optimization, I just took all of the flags
-# from O3 here: https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html and put them
-# in individually. A few were removed due to compile or runtime errors. In addition, I
-# added some flags applied by default from the conda gfortran compiler (-ftree-vectorize 
-# -fPIC -fno-plt -ffunction-sections). I also added the march flag for the google 
-# machines we are using (broadwell currently), and dropped fstack-protector-strong from
-# the list of conda-based flags, as it was throwing segfaults.
 ENV CLAW_FFLAGS="-DNETCDF -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include \
-  -fauto-inc-dec -fbranch-count-reg -fcombine-stack-adjustments -fcompare-elim \
-  -fcprop-registers -fdce -fdefer-pop -fdse -fforward-propagate \
-  -fguess-branch-probability -fif-conversion -fif-conversion2 \
-  -finline-functions-called-once -fipa-profile -fipa-pure-const -fipa-reference \
-  -fmerge-constants -fmove-loop-invariants -fomit-frame-pointer -freorder-blocks \
-  -fshrink-wrap -fshrink-wrap-separate -fsplit-wide-types -fssa-backprop -fssa-phiopt \
-  -ftree-bit-ccp -ftree-ccp -ftree-ch -ftree-coalesce-vars -ftree-copy-prop -ftree-dce \
-  -ftree-dominator-opts -ftree-dse -ftree-forwprop -ftree-fre -ftree-phiprop \
-  -ftree-pta -ftree-scev-cprop -ftree-sink -ftree-slsr -ftree-sra -ftree-ter \
-  -funit-at-a-time -freorder-blocks-algorithm=stc -freorder-blocks-and-partition \
-  -freorder-functions -frerun-cse-after-loop -fschedule-insns -fschedule-insns2 \
-  -fsched-interblock -fsched-spec -fstore-merging -fstrict-aliasing -fthread-jumps \
-  -ftree-builtin-call-dce -ftree-pre -ftree-switch-conversion  -ftree-tail-merge \
-  -ftree-vrp -fgcse  -fgcse-lm  -fhoist-adjacent-loads -finline-functions \
-  -finline-small-functions -findirect-inlining -fipa-bit-cp  -fipa-cp  -fipa-icf \
-  -fipa-ra  -fipa-sra  -fipa-vrp -fisolate-erroneous-paths-dereference -flra-remat \
-  -foptimize-sibling-calls -foptimize-strlen -fpartial-inlining -fpeephole2 \
-  -falign-functions -falign-jumps -falign-labels  -falign-loops -fcaller-saves \
-  -fcode-hoisting -fcrossjumping -fcse-follow-jumps  -fcse-skip-blocks \
-  -fdelete-null-pointer-checks -fdevirtualize  -fdevirtualize-speculatively \
-  -fexpensive-optimizations -fgcse-after-reload -floop-unroll-and-jam -fpeel-loops \
-  -fpredictive-commoning -fsplit-loops -fsplit-paths -ftree-loop-distribution \
-  -ftree-loop-vectorize -ftree-partial-pre -ftree-slp-vectorize -funswitch-loops \
-  -fvect-cost-model -fvect-cost-model=dynamic -march=broadwell -ftree-vectorize -fPIC \
-  -fno-plt -ffunction-sections"
+  -O3 -march=broadwell -ftree-vectorize -fPIC -fno-plt -ffunction-sections \
+  -fstack-protector-strong"
 
 # need to change shell in order for source command to work
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]

From 917fbc3cb90e5508921cddd6833a71ca33344df2 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 31 Jul 2020 14:58:43 -0700
Subject: [PATCH 154/224] bump clawpack commit

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 94cdc30..03e6ccc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=6485e04a0daee160612e7fabd91c440835901903
+    - CLAWPACK_COMMIT=5d8871aeaed995fc8c25e1d5793b8b530c0cda29
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From 0c103fa2b7f74d1eacc9bf15cb779b6939ae33aa Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 31 Jul 2020 14:59:59 -0700
Subject: [PATCH 155/224] fix dockerfile typo

---
 worker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/worker/Dockerfile b/worker/Dockerfile
index c3f607c..afed281 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -40,7 +40,7 @@ RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py38_${MINICONDA
     rm Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
     conda config --system --prepend channels conda-forge && \
     conda config --system --set channel_priority strict && \
-    conda update -n base --yes conda \
+    conda update -n base --yes conda && \
     conda update --all --yes
 ###########
 

From 103421c432835cacd1458ada94f16aaa55b19851 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 31 Jul 2020 20:15:44 -0700
Subject: [PATCH 156/224] bump geoclaw commit

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 03e6ccc..432762e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=5d8871aeaed995fc8c25e1d5793b8b530c0cda29
+    - CLAWPACK_COMMIT=32a7991daa7242558ebcbbc01a4bd3b27f4f17a5
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From 7e1999b4d95a719cbe96e66c02ec5ee93a880e13 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 3 Aug 2020 13:45:26 -0700
Subject: [PATCH 157/224] fix worker workdir

---
 worker/Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/worker/Dockerfile b/worker/Dockerfile
index afed281..49a527b 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -93,4 +93,6 @@ ENV OMP_NUM_THREADS=1
 ENV MKL_NUM_THREADS=1
 ENV OPENBLAS_NUM_THREADS=1
 
+WORKDIR /
+
 ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]

From 8a8ae425ace14045a24e9623172c0a55a8eca523 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Wed, 5 Aug 2020 16:20:26 -0700
Subject: [PATCH 158/224] set default worker image to pytc-image

---
 .travis.yml                  | 10 ----------
 notebook/worker-template.yml |  2 +-
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 432762e..0215be1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,16 +9,6 @@ env:
     - CLAWPACK_COMMIT=32a7991daa7242558ebcbbc01a4bd3b27f4f17a5
 
 install:
-- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
-    sed -i.bak \
-      's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' \
-      notebook/worker-template.yml; 
-  else
-    sed -i.bak \
-      's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' \
-      notebook/worker-template.yml; fi"
-- "rm notebook/worker-template.yml.bak"
-- "cat notebook/worker-template.yml | grep image:"
 - "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \
     $IMAGE_NAME/shared_resources"
 - "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \
diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml
index a8990c2..858c456 100755
--- a/notebook/worker-template.yml
+++ b/notebook/worker-template.yml
@@ -14,7 +14,7 @@ spec:
     env:
       - name: GCSFUSE_BUCKET
         value: rhg-data
-    image: rhodium/worker:WILL_BE_REPLACED_BY_TRAVIS
+    image: gcr.io/rhg-project-1/pytc-image-devbase:latest
     name: dask-worker
     securityContext:
       capabilities:

From f695977e82090600ef19da918a20ef1f9b519ea0 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Wed, 5 Aug 2020 21:27:48 -0700
Subject: [PATCH 159/224] try sed for worker templte update

---
 .travis.yml                  | 11 +++++++++++
 notebook/worker-template.yml |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 0215be1..f400ad5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,17 @@ env:
     - CLAWPACK_COMMIT=32a7991daa7242558ebcbbc01a4bd3b27f4f17a5
 
 install:
+- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
+    sed -i.bak \
+      's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' \
+      notebook/worker-template.yml; 
+  else
+    sed -i.bak \
+      's/image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:.*/\
+      image: \gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_TAG\"'/' \
+      notebook/worker-template.yml; fi"
+- "rm notebook/worker-template.yml.bak"
+- "cat notebook/worker-template.yml | grep image:"
 - "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \
     $IMAGE_NAME/shared_resources"
 - "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \
diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml
index 858c456..ed112c9 100755
--- a/notebook/worker-template.yml
+++ b/notebook/worker-template.yml
@@ -14,7 +14,7 @@ spec:
     env:
       - name: GCSFUSE_BUCKET
         value: rhg-data
-    image: gcr.io/rhg-project-1/pytc-image-devbase:latest
+    image: gcr.io/rhg-project-1/pytc-image-devbase:WILL_BE_REPLACED_BY_TRAVIS
     name: dask-worker
     securityContext:
       capabilities:

From a200f01d4e216dc18e1de4b9ebfe654466d90c7a Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sat, 8 Aug 2020 13:32:04 -0700
Subject: [PATCH 160/224] bump clawpack commit

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f400ad5..7b9eb3d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=32a7991daa7242558ebcbbc01a4bd3b27f4f17a5
+    - CLAWPACK_COMMIT=c8d07a22bb5c86c39e5f7730a15ac1585f29e828
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From d2a06755b0db3031838a59ac5aa83377029d0d53 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sat, 8 Aug 2020 13:34:39 -0700
Subject: [PATCH 161/224] fix travis typo

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7b9eb3d..4c1d1b8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ install:
   else
     sed -i.bak \
       's/image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:.*/\
-      image: \gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_TAG\"'/' \
+      image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_TAG\"'/' \
       notebook/worker-template.yml; fi"
 - "rm notebook/worker-template.yml.bak"
 - "cat notebook/worker-template.yml | grep image:"

From 88f8183596c56bb3fbb65126bb89da675a602198 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sat, 8 Aug 2020 13:51:19 -0700
Subject: [PATCH 162/224] bump clawpack again

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 4c1d1b8..0af8280 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=c8d07a22bb5c86c39e5f7730a15ac1585f29e828
+    - CLAWPACK_COMMIT=5b526918cee7f3241fbc60de5bee45ffeb2b120b
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From 58e47877a6bdb78bd800a05999227b0bfb0dac88 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sat, 8 Aug 2020 21:22:00 -0700
Subject: [PATCH 163/224] unpin notebook packages

---
 notebook/notebook_environment.yml | 47 +++++++++++++++----------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 31f9a3a..e6368f8 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -2,33 +2,32 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - black=19.10b0=py_4
-  - bump2version=1.0.0=pyh9f0ad1d_0
-  - coverage=5.2.1=py38h1e0a361_0
-  - dask-labextension=2.0.2=py_0
-  - flake8=3.8.3=py_1
-  - ipdb=0.13.3=pyh9f0ad1d_0
-  - ipyleaflet=0.13.3=pyh9f0ad1d_0
-  - ipympl=0.5.7=pyh9f0ad1d_1
-  - ipypublish=0.10.11=py38h32f6830_0
-  - isort=5.2.2=py38h32f6830_0
+  - black
+  - bump2version
+  - coverage
+  - dask-labextension
+  - flake8
+  - ipdb
+  - ipyleaflet
+  - ipympl
+  - ipypublish
+  - isort
 # need jupyterhub version to match that running on the hub
   - jupyterhub=1.0.0=py38_0 # pinkeep: jupyterhub=1.0.0
-  - jupyterlab_code_formatter=1.3.5=py_0
-  - nano=2.9.8=hddfc1eb_1001
-  - nbdime=2.0.0=py_1
-  - nb_conda_kernels=2.2.3=py38_0
+  - jupyterlab_code_formatter
+  - nano
+  - nbdime
+  - nb_conda_kernels
   - octave_kernel=0.32.0=pyh9f0ad1d_0
   - oct2py=5.0.4=py_0
-  - openssh=8.3p1=h5957347_0
-  - papermill=2.1.2=py38h32f6830_0
-  - pip=20.2=py_0
-  - pytest=6.0.1=py38h32f6830_0
-  - pytest-cov=2.10.0=pyh9f0ad1d_0
-  - pytest-runner=5.2=py_0
-  - python-graphviz=0.14.1=pyh9f0ad1d_0
-  - sidecar=0.4.0=pyh9f0ad1d_0
-  - sphinx_rtd_theme=0.5.0=pyh9f0ad1d_0
-  - tox=3.18.1=py38h32f6830_0
+  - openssh
+  - papermill
+  - pip
+  - pytest
+  - pytest-cov
+  - python-graphviz
+  - sidecar
+  - sphinx_rtd_theme
+  - tox
   - pip:
     - black_nbconvert

From f54e71bd738110a2f86278160e9de72723c35902 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 9 Aug 2020 07:54:18 -0700
Subject: [PATCH 164/224] bump blas, oct2py, scikit-learn, dask, others

---
 notebook/notebook_environment.yml     |  2 +-
 octave-worker/octave_environment.yml  |  2 +-
 shared_resources/base_environment.yml | 16 ++++++++--------
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index e6368f8..8ccc849 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -19,7 +19,7 @@ dependencies:
   - nbdime
   - nb_conda_kernels
   - octave_kernel=0.32.0=pyh9f0ad1d_0
-  - oct2py=5.0.4=py_0
+  - oct2py=5.2.0=pyh9f0ad1d_0
   - openssh
   - papermill
   - pip
diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml
index 7585324..78742ad 100755
--- a/octave-worker/octave_environment.yml
+++ b/octave-worker/octave_environment.yml
@@ -3,4 +3,4 @@ channels:
   - conda-forge
 dependencies:
   - octave_kernel=0.32.0=pyh9f0ad1d_0
-  - oct2py=5.0.4=py_0
+  - oct2py=5.2.0=pyh9f0ad1d_0
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 71fc2b8..443f8ff 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -2,22 +2,22 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4=4.9.1=py38h32f6830_0
-  - blas=2.14=openblas
+  - blas=2.17=openblas
   - bottleneck=1.3.2=py38h8790de6_1
-  - boto3=1.14.32=pyh9f0ad1d_0
-  - bqplot=0.12.14=pyh9f0ad1d_0
+  - boto3=1.14.38=pyh9f0ad1d_0
+  - bqplot=0.12.15=pyh9f0ad1d_0
   - cachecontrol=0.12.6=py_0
   - cartopy=0.18.0=py38h172510d_0
   - cftime=1.2.1=py38h8790de6_0
   - click=7.1.2=pyh9f0ad1d_0
   - compilers=1.1.1=0
-  - dask=2.21.0=py_0
+  - dask=2.22.0=py_0
   - dask-glm=0.2.0=py_1
   - dask-kubernetes=0.10.1=py_0
   - dask-ml=1.6.0=py_0
   - datashader=0.10.0=py_0
   - descartes=1.1.0=py_4
-  - distributed=2.21.0=py38h32f6830_0
+  - distributed=2.22.0=py38h32f6830_0
   - dropbox=10.1.1=pyh9f0ad1d_0
   - esmpy=8.0.1=nompi_py38hbd9704b_0
   - fastparquet=0.4.1=py38h8790de6_0
@@ -29,8 +29,8 @@ dependencies:
   - geopandas=0.8.1=py_0
   - geotiff=1.6.0=h05acad5_0
   - geoviews=1.8.1=py_0
-  - git=2.28.0=pl526h5e3e691_0
-  - gitpython=3.1.3=py_0
+  - git=2.28.0=pl526h5e3e691_1
+  - gitpython=3.1.7=py_0
   - google-cloud-container=0.3.0=py38_0
   - google-cloud-storage=1.30.0=pyh9f0ad1d_0
   - holoviews=1.13.3=pyh9f0ad1d_0
@@ -64,7 +64,7 @@ dependencies:
   - regionmask=0.6.0=py_0
   - rtree=0.9.4=py38h08f867b_1
   - scikit-image=0.17.2=py38hcb8c335_1
-  - scikit-learn=0.23.1=py38h3a94b23_0
+  - scikit-learn=0.23.2=py38hee58b96_0
   - scipy=1.5.2=py38h8c5af15_0
   - seaborn=0.10.1=1
   - selenium=3.141.0=py38h1e0a361_1001 # for geoviews

From 3b659a26d1a1c984c9fe6d6c5e9411c001aa6b33 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 9 Aug 2020 07:54:26 -0700
Subject: [PATCH 165/224] bump clawpack commit

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 0af8280..c7303f0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=5b526918cee7f3241fbc60de5bee45ffeb2b120b
+    - CLAWPACK_COMMIT=1a7eed585c32fa66ef59c19a60d6e649fd067bea
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From 7ee6e61f28df6db5fb4de4e7056ed3900fd91dea Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 9 Aug 2020 11:26:51 -0700
Subject: [PATCH 166/224] configure nbdime

---
 notebook/Dockerfile         | 7 +++++++
 notebook/nbdime_config.json | 5 +++++
 2 files changed, 12 insertions(+)
 create mode 100644 notebook/nbdime_config.json

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index e62337d..bbef82e 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -79,6 +79,13 @@ RUN jupyter labextension update --all && \
       plotlywidget && \
     jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
 
+## configure nbdime
+RUN source /opt/conda/etc/profile.d/conda.sh \
+  && conda activate \
+  && nbdime config-git --enable --system
+# makes sure that the web-tools allow accessible IP
+COPY nbdime_config.json /opt/conda/etc/jupyter
+
 ##################
 ## clawpack added for coastal-specific image
 ##################
diff --git a/notebook/nbdime_config.json b/notebook/nbdime_config.json
new file mode 100644
index 0000000..6bbdaca
--- /dev/null
+++ b/notebook/nbdime_config.json
@@ -0,0 +1,5 @@
+{
+    "WebTool": {
+        "ip": "*"
+    }
+}
\ No newline at end of file

From 5aa7842922a82911af682301d27c2f03e74aa52a Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 9 Aug 2020 11:33:25 -0700
Subject: [PATCH 167/224] bump vdatum to 4.1

---
 shared_resources/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared_resources/common.sh b/shared_resources/common.sh
index 69f784f..c130c3a 100755
--- a/shared_resources/common.sh
+++ b/shared_resources/common.sh
@@ -38,7 +38,7 @@ wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/clou
 chmod +x /usr/bin/cloud_sql_proxy
 
 # download vdatum
-wget https://vdatum.noaa.gov/download/data/vdatum_v4.0.1.zip -O /usr/bin/vdatum.zip
+wget https://vdatum.noaa.gov/download/data/vdatum_v4.1.zip -O /usr/bin/vdatum.zip
 wget https://vdatum.noaa.gov/download/data/vdatum_EGM1996.zip -O /usr/bin/vdatum_EGM1996.zip
 unzip /usr/bin/vdatum.zip -d /usr/bin
 unzip -o /usr/bin/vdatum_EGM1996.zip -d /usr/bin

From c77d8644f36cab359854af0b2d38be8d0e38e74d Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Sun, 9 Aug 2020 18:59:37 -0700
Subject: [PATCH 168/224] fix auto worker template

---
 .travis.yml                  | 3 ++-
 notebook/worker-template.yml | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index c7303f0..1eb60d1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,8 @@ env:
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
     sed -i.bak \
-      's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' \
+      's/image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:.*/\
+      image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_COMMIT\"'/' \
       notebook/worker-template.yml; 
   else
     sed -i.bak \
diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml
index ed112c9..2e6915b 100755
--- a/notebook/worker-template.yml
+++ b/notebook/worker-template.yml
@@ -15,6 +15,7 @@ spec:
       - name: GCSFUSE_BUCKET
         value: rhg-data
     image: gcr.io/rhg-project-1/pytc-image-devbase:WILL_BE_REPLACED_BY_TRAVIS
+    imagePullPolicy: Always
     name: dask-worker
     securityContext:
       capabilities:

From 0828fd64307fbfa9f1df48b6f2f4479f6574f86a Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Mon, 10 Aug 2020 16:01:54 -0700
Subject: [PATCH 169/224] add us package

---
 shared_resources/base_environment.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 443f8ff..66ba76a 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -38,6 +38,7 @@ dependencies:
   - h5netcdf=0.8.1=py_0
   - intake-esm=2020.6.11=py_0
   - jedi=0.17.2=py38h32f6830_0
+  - jellyfish=0.7.2=py38h1e0a361_1 # needed for "us" package
   - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
   - kubernetes=1.18.6=0
   - libmatio=1.5.12=hf54d351_1004
@@ -84,4 +85,5 @@ dependencies:
     - parameterize-jobs==0.1.1
     - rhg_compute_tools==0.2.2
     - git+https://github.com/maritimeplanning/pytides.git@a8397a0e12645d22c62612385d18e63c4d67d6fa#egg=pytides
+    - us==2.0.2 # installing from pip b/c conda thinks we need py 3.7
 name: base

From 2370cba3d2fb81438d0a258284b743c23c382a3f Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Tue, 11 Aug 2020 12:51:45 -0700
Subject: [PATCH 170/224] install openpyxl and lxml

---
 shared_resources/base_environment.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 66ba76a..da1c522 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -42,6 +42,7 @@ dependencies:
   - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
   - kubernetes=1.18.6=0
   - libmatio=1.5.12=hf54d351_1004
+  - lxml=4.5.2=py38hbb43d70_0
   - lz4=3.1.0=py38h66f7c9e_0
   - make=4.3=h516909a_0
   - matplotlib=3.3.0=1
@@ -50,6 +51,7 @@ dependencies:
   - netcdf4=1.5.4=nompi_py38hfd55d45_100
   - nose=1.3.7=py38h32f6830_1004
   - numba=0.50.1=py38hcb8c335_1
+  - openpyxl=3.0.4=py_0
   - pandas=1.1.0=py38h950e882_0
   - phantomjs=2.1.1=1 # for geoviews
   - pillow=7.2.0=py38h9776b28_1

From f941670d5a6e40e8da66d65fbfe4c8e4f90c4107 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 14 Aug 2020 09:50:49 -0700
Subject: [PATCH 171/224] try out elyra

---
 notebook/Dockerfile                   | 3 +++
 notebook/notebook_environment.yml     | 1 +
 shared_resources/base_environment.yml | 6 +++---
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index bbef82e..d5de12a 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -54,6 +54,9 @@ RUN conda update -n base conda
 #  update environemnt with common packages across worker and nb
 RUN conda env update -f /opt/conda/specs/base_environment.yml
 
+# need access to settings/page_config.json for elyra
+RUN /tempdir/fix-permissions.sh /opt/conda/share/jupyter/lab/settings
+
 # update environment with nb-specific packages
 RUN conda env update -f /opt/conda/specs/notebook_environment.yml
 
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 8ccc849..e7bf6e4 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -31,3 +31,4 @@ dependencies:
   - tox
   - pip:
     - black_nbconvert
+    - elyra
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index da1c522..7cfaecb 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -4,7 +4,7 @@ dependencies:
   - beautifulsoup4=4.9.1=py38h32f6830_0
   - blas=2.17=openblas
   - bottleneck=1.3.2=py38h8790de6_1
-  - boto3=1.14.38=pyh9f0ad1d_0
+  - boto3=1.14.42=pyh9f0ad1d_0
   - bqplot=0.12.15=pyh9f0ad1d_0
   - cachecontrol=0.12.6=py_0
   - cartopy=0.18.0=py38h172510d_0
@@ -40,8 +40,8 @@ dependencies:
   - jedi=0.17.2=py38h32f6830_0
   - jellyfish=0.7.2=py38h1e0a361_1 # needed for "us" package
   - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
-  - kubernetes=1.18.6=0
-  - libmatio=1.5.12=hf54d351_1004
+  - kubernetes=1.18.8=0
+  - libmatio=1.5.17=hf54d351_0
   - lxml=4.5.2=py38hbb43d70_0
   - lz4=3.1.0=py38h66f7c9e_0
   - make=4.3=h516909a_0

From 6f9d40f06e067fa1bb0dcffdb67c48b0b9153ab4 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 14 Aug 2020 12:04:21 -0700
Subject: [PATCH 172/224] bump notebook image

---
 notebook/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index d5de12a..d9a55f7 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook@sha256:b5abe43c6d312b5a7dd63e33f2291df52cddfc1ac15547542ac8b0990a30b0e5
+FROM jupyter/base-notebook@sha256:6e276c402aa19799798b84b46e0e08fc03be193de9a81490158ca4d9026b8cae
 ARG DEBIAN_FRONTEND=noninteractive
 
 ## needed to make sure things with cython compile correctly
@@ -55,7 +55,9 @@ RUN conda update -n base conda
 RUN conda env update -f /opt/conda/specs/base_environment.yml
 
 # need access to settings/page_config.json for elyra
+USER root
 RUN /tempdir/fix-permissions.sh /opt/conda/share/jupyter/lab/settings
+USER $NB_USER
 
 # update environment with nb-specific packages
 RUN conda env update -f /opt/conda/specs/notebook_environment.yml

From 67197cada9d4db8d608eab8429d5a713c5109848 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 14 Aug 2020 12:49:00 -0700
Subject: [PATCH 173/224] add travis_wait

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 1eb60d1..98b3991 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,7 +29,7 @@ install:
 
 
 script:
-- docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
+- travis_wait docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \
     ../octave-worker; fi"

From 70f8ec009697c5083f263b120f7a6ea3dd4dee0b Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 14 Aug 2020 13:58:44 -0700
Subject: [PATCH 174/224] try w/o travis_wait again

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 98b3991..1eb60d1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,7 +29,7 @@ install:
 
 
 script:
-- travis_wait docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
+- docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \
     ../octave-worker; fi"

From 74ade4ebf9dd45043fc1193cabdb3474350f43e2 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 14 Aug 2020 15:18:50 -0700
Subject: [PATCH 175/224] try longer travis_wait

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 1eb60d1..df223c3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,7 +29,7 @@ install:
 
 
 script:
-- docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
+- travis_wait 30 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \
     ../octave-worker; fi"

From 0767979da0117cf245ce7ecb0bc5023d15619173 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <bolliger@berkeley.edu>
Date: Fri, 14 Aug 2020 18:21:21 -0700
Subject: [PATCH 176/224] try 90 min travis wait

---
 .travis.yml         | 2 +-
 notebook/Dockerfile | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index df223c3..281e0a0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,7 +29,7 @@ install:
 
 
 script:
-- travis_wait 30 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
+- travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \
     ../octave-worker; fi"
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index d9a55f7..a282c5c 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -71,18 +71,17 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
     /opt/conda/bin/matio_73to5
 
 ## Set up jupyter lab extensions
-RUN jupyter labextension update --all && \
-    jupyter labextension install \
+RUN jupyter labextension update --all 
+RUN jupyter labextension install \
       @bokeh/jupyter_bokeh \
-      @jupyterlab/toc \
       @jupyter-widgets/jupyterlab-manager \
       @lckr/jupyterlab_variableinspector \
       @pyviz/jupyterlab_pyviz \
       @ryantam626/jupyterlab_code_formatter \
       dask-labextension \
       jupyterlab-plotly \
-      plotlywidget && \
-    jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
+      plotlywidget
+RUN jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
 
 ## configure nbdime
 RUN source /opt/conda/etc/profile.d/conda.sh \

From 84d336a2d72fc9d912e8817eb507156a9c110521 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ianbolliger@rhgmac-ibolliger.local>
Date: Wed, 19 Aug 2020 12:53:32 -0700
Subject: [PATCH 177/224] add mypy to nb

---
 notebook/notebook_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index e7bf6e4..b3777e8 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -15,6 +15,7 @@ dependencies:
 # need jupyterhub version to match that running on the hub
   - jupyterhub=1.0.0=py38_0 # pinkeep: jupyterhub=1.0.0
   - jupyterlab_code_formatter
+  - mypy
   - nano
   - nbdime
   - nb_conda_kernels

From 334ed8486605d1efc0d783c5a6c67dfe83c4f5ad Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ianbolliger@rhgmac-ibolliger.local>
Date: Thu, 20 Aug 2020 15:43:01 -0700
Subject: [PATCH 178/224] bump clawpack commit

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 281e0a0..47b6bd8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=1a7eed585c32fa66ef59c19a60d6e649fd067bea
+    - CLAWPACK_COMMIT=e01b6b3e8c519da40b76a74e60342767b3cccab8
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From b7050d68d2b875205d60db874e6361820a28d802 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Sun, 23 Aug 2020 11:15:41 -0700
Subject: [PATCH 179/224] update pin to rm r_env

---
 pin.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pin.py b/pin.py
index b2b426e..c010af1 100644
--- a/pin.py
+++ b/pin.py
@@ -28,8 +28,7 @@
 SPEC_FILES = [
             ('shared_resources/base_environment.yml', 'base'),
             ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base'),
-            ('notebook/r_environment.yml', 'r')]
+            ('octave-worker/octave_environment.yml', 'base')]
 
 
 def get_versions_in_current_environment(envname='base'):

From 0cd3618b1f0063bb58c8263fb2a78e529f1b5f2b Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Sun, 23 Aug 2020 21:36:58 +0000
Subject: [PATCH 180/224] add gnuplot for octave and bump packages

---
 .gitignore                            |  1 +
 notebook/Dockerfile                   |  2 +-
 pin.py                                |  7 +++--
 shared_resources/base_environment.yml | 40 +++++++++++++--------------
 4 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/.gitignore b/.gitignore
index 765dc93..148d7d7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ worker-config.yml
 __pycache__
 .pytest_cache
 .vscode/settings.json
+.devcontainer.json
\ No newline at end of file
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index a282c5c..de665ac 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -37,7 +37,7 @@ RUN sudo chown -R $NB_USER /gcs
 ## more apt-get
 RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
   vim less man locate kmod dialog \
-  octave octave-optim && \
+  octave octave-optim gnuplot fonts-freefont-otf ghostscript && \
   sudo apt-get clean
 
 
diff --git a/pin.py b/pin.py
index c010af1..901a2ac 100644
--- a/pin.py
+++ b/pin.py
@@ -26,9 +26,10 @@
 
 
 SPEC_FILES = [
-            ('shared_resources/base_environment.yml', 'base'),
-            ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base')]
+    ('shared_resources/base_environment.yml', 'base'),
+    ('notebook/notebook_environment.yml', 'base'),
+    ('octave-worker/octave_environment.yml', 'base')
+]
 
 
 def get_versions_in_current_environment(envname='base'):
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 7cfaecb..f9db5f7 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -1,30 +1,30 @@
 channels:
   - conda-forge
 dependencies:
-  - beautifulsoup4=4.9.1=py38h32f6830_0
-  - blas=2.17=openblas
+  - beautifulsoup4=4.9.1=py_1
+  - blas=2.14=openblas
   - bottleneck=1.3.2=py38h8790de6_1
-  - boto3=1.14.42=pyh9f0ad1d_0
-  - bqplot=0.12.15=pyh9f0ad1d_0
+  - boto3=1.14.47=pyh9f0ad1d_0
+  - bqplot=0.12.16=pyh9f0ad1d_0
   - cachecontrol=0.12.6=py_0
   - cartopy=0.18.0=py38h172510d_0
   - cftime=1.2.1=py38h8790de6_0
   - click=7.1.2=pyh9f0ad1d_0
   - compilers=1.1.1=0
-  - dask=2.22.0=py_0
+  - dask=2.23.0=py_0
   - dask-glm=0.2.0=py_1
   - dask-kubernetes=0.10.1=py_0
   - dask-ml=1.6.0=py_0
-  - datashader=0.10.0=py_0
+  - datashader=0.11.1=pyh9f0ad1d_0
   - descartes=1.1.0=py_4
-  - distributed=2.22.0=py38h32f6830_0
+  - distributed=2.24.0=py38h32f6830_0
   - dropbox=10.1.1=pyh9f0ad1d_0
   - esmpy=8.0.1=nompi_py38hbd9704b_0
   - fastparquet=0.4.1=py38h8790de6_0
   - fiona=1.8.13=py38h033e0f6_1
   - fusepy=3.0.1=py_0
   - fuzzywuzzy=0.17.0=py_0
-  - gcsfs=0.6.2=py_0
+  - gcsfs=0.7.0=py_0
   - gdal=3.0.4=py38h172510d_10
   - geopandas=0.8.1=py_0
   - geotiff=1.6.0=h05acad5_0
@@ -36,7 +36,7 @@ dependencies:
   - holoviews=1.13.3=pyh9f0ad1d_0
   - hvplot=0.6.0=pyh9f0ad1d_0
   - h5netcdf=0.8.1=py_0
-  - intake-esm=2020.6.11=py_0
+  - intake-esm=2020.8.15=py_0
   - jedi=0.17.2=py38h32f6830_0
   - jellyfish=0.7.2=py38h1e0a361_1 # needed for "us" package
   - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
@@ -45,34 +45,34 @@ dependencies:
   - lxml=4.5.2=py38hbb43d70_0
   - lz4=3.1.0=py38h66f7c9e_0
   - make=4.3=h516909a_0
-  - matplotlib=3.3.0=1
+  - matplotlib=3.3.1=1
   - nc-time-axis=1.2.0=py_1
   - netcdf-fortran=4.5.3=nompi_hfef6a68_100
-  - netcdf4=1.5.4=nompi_py38hfd55d45_100
+  - netcdf4=1.5.4=nompi_py38hec8b9af_101
   - nose=1.3.7=py38h32f6830_1004
-  - numba=0.50.1=py38hcb8c335_1
-  - openpyxl=3.0.4=py_0
+  - numba=0.51.0=py38hc5bc63f_0
+  - openpyxl=3.0.5=py_0
   - pandas=1.1.0=py38h950e882_0
   - phantomjs=2.1.1=1 # for geoviews
   - pillow=7.2.0=py38h9776b28_1
-  - pip=20.2=py_0
+  - pip=20.2.2=py_0
   - plotly=4.9.0=pyh9f0ad1d_0
   - pygeos=0.7.1=py38hd168ffb_0
   - pyinterp=0.3.2=mkl_py38hefd7b46_102
   - pyshp=2.1.0=py_0
-  - python=3.8.5=h4d41432_2_cpython
+  - python=3.8.5=h1103e12_5_cpython
   - python-snappy=0.5.4=py38h7cfaab3_1
   - pyyaml=5.3.1=py38h1e0a361_0
   - rasterio=1.1.5=py38h033e0f6_1
-  - regionmask=0.6.0=py_0
+  - regionmask=0.6.1=py_0
   - rtree=0.9.4=py38h08f867b_1
   - scikit-image=0.17.2=py38hcb8c335_1
   - scikit-learn=0.23.2=py38hee58b96_0
   - scipy=1.5.2=py38h8c5af15_0
   - seaborn=0.10.1=1
   - selenium=3.141.0=py38h1e0a361_1001 # for geoviews
-  - shapely=1.7.0=py38hd168ffb_3
-  - sparse=0.10.0=py_0
+  - shapely=1.7.1=py38hc7361b7_0
+  - sparse=0.11.0=py_0
   - statsmodels=0.11.1=py38h1e0a361_2
   - tini=0.18.0=h14c3975_1001
   - xarray=0.16.0=py_0
@@ -82,8 +82,8 @@ dependencies:
   - zarr=2.4.0=py_0
   - zeromq=4.3.2=he1b5a44_3
   - pip:
-    - climate-toolbox==0.1.5 
-    - noaa-coops==0.1.5
+    - climate-toolbox==0.1.5
+    - noaa-coops==0.1.8
     - parameterize-jobs==0.1.1
     - rhg_compute_tools==0.2.2
     - git+https://github.com/maritimeplanning/pytides.git@a8397a0e12645d22c62612385d18e63c4d67d6fa#egg=pytides

From e9c826304f8b244f7a942b4bda7f2d3c1c24824b Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 26 Aug 2020 12:38:48 -0700
Subject: [PATCH 181/224] bump blas, dask, and pandas

---
 shared_resources/base_environment.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index f9db5f7..40e0045 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -2,16 +2,16 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4=4.9.1=py_1
-  - blas=2.14=openblas
+  - blas=2.17=openblas
   - bottleneck=1.3.2=py38h8790de6_1
-  - boto3=1.14.47=pyh9f0ad1d_0
+  - boto3=1.14.48=pyh9f0ad1d_0
   - bqplot=0.12.16=pyh9f0ad1d_0
   - cachecontrol=0.12.6=py_0
   - cartopy=0.18.0=py38h172510d_0
   - cftime=1.2.1=py38h8790de6_0
   - click=7.1.2=pyh9f0ad1d_0
   - compilers=1.1.1=0
-  - dask=2.23.0=py_0
+  - dask=2.24.0=py_0
   - dask-glm=0.2.0=py_1
   - dask-kubernetes=0.10.1=py_0
   - dask-ml=1.6.0=py_0
@@ -48,11 +48,11 @@ dependencies:
   - matplotlib=3.3.1=1
   - nc-time-axis=1.2.0=py_1
   - netcdf-fortran=4.5.3=nompi_hfef6a68_100
-  - netcdf4=1.5.4=nompi_py38hec8b9af_101
+  - netcdf4=1.5.4=nompi_py38hec8b9af_102
   - nose=1.3.7=py38h32f6830_1004
   - numba=0.51.0=py38hc5bc63f_0
   - openpyxl=3.0.5=py_0
-  - pandas=1.1.0=py38h950e882_0
+  - pandas=1.1.1=py38h950e882_0
   - phantomjs=2.1.1=1 # for geoviews
   - pillow=7.2.0=py38h9776b28_1
   - pip=20.2.2=py_0

From c45059d4e8f3e6d4d899be83d277fafc52d1f7a5 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 26 Aug 2020 16:39:30 -0700
Subject: [PATCH 182/224] add jupyterlab spec so that it updates

---
 notebook/notebook_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index b3777e8..ac7861c 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -14,6 +14,7 @@ dependencies:
   - isort
 # need jupyterhub version to match that running on the hub
   - jupyterhub=1.0.0=py38_0 # pinkeep: jupyterhub=1.0.0
+  - jupyterlab
   - jupyterlab_code_formatter
   - mypy
   - nano

From bc2eff808d837023d0085101739054348fbc8b08 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Fri, 28 Aug 2020 09:42:58 -0700
Subject: [PATCH 183/224] add xarray-extras and esmpy w/ mpi to enable xesmf

---
 shared_resources/base_environment.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 40e0045..44d7951 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -19,7 +19,7 @@ dependencies:
   - descartes=1.1.0=py_4
   - distributed=2.24.0=py38h32f6830_0
   - dropbox=10.1.1=pyh9f0ad1d_0
-  - esmpy=8.0.1=nompi_py38hbd9704b_0
+  - esmpy=8.0.1=mpi_openmpi_py38h51f2404_0
   - fastparquet=0.4.1=py38h8790de6_0
   - fiona=1.8.13=py38h033e0f6_1
   - fusepy=3.0.1=py_0
@@ -47,7 +47,7 @@ dependencies:
   - make=4.3=h516909a_0
   - matplotlib=3.3.1=1
   - nc-time-axis=1.2.0=py_1
-  - netcdf-fortran=4.5.3=nompi_hfef6a68_100
+  - netcdf-fortran=4.5.3=mpi_openmpi_h2cce2cb_0
   - netcdf4=1.5.4=nompi_py38hec8b9af_102
   - nose=1.3.7=py38h32f6830_1004
   - numba=0.51.0=py38hc5bc63f_0
@@ -76,6 +76,7 @@ dependencies:
   - statsmodels=0.11.1=py38h1e0a361_2
   - tini=0.18.0=h14c3975_1001
   - xarray=0.16.0=py_0
+  - xarray-extras=0.4.2=py38h516909a_1000
   - xesmf=0.3.0=py_0
   - xhistogram=0.1.1=py_0
   - xlrd=1.2.0=pyh9f0ad1d_1

From 0077149dcd349479fb04a7e68f065ee3d459fda3 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Sat, 29 Aug 2020 15:47:25 -0700
Subject: [PATCH 184/224] bump dask, numba, statsmodels, gcs, python build

---
 shared_resources/base_environment.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 44d7951..965423f 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -4,20 +4,20 @@ dependencies:
   - beautifulsoup4=4.9.1=py_1
   - blas=2.17=openblas
   - bottleneck=1.3.2=py38h8790de6_1
-  - boto3=1.14.48=pyh9f0ad1d_0
+  - boto3=1.14.51=pyh9f0ad1d_0
   - bqplot=0.12.16=pyh9f0ad1d_0
   - cachecontrol=0.12.6=py_0
   - cartopy=0.18.0=py38h172510d_0
   - cftime=1.2.1=py38h8790de6_0
   - click=7.1.2=pyh9f0ad1d_0
   - compilers=1.1.1=0
-  - dask=2.24.0=py_0
+  - dask=2.25.0=py_0
   - dask-glm=0.2.0=py_1
   - dask-kubernetes=0.10.1=py_0
   - dask-ml=1.6.0=py_0
   - datashader=0.11.1=pyh9f0ad1d_0
   - descartes=1.1.0=py_4
-  - distributed=2.24.0=py38h32f6830_0
+  - distributed=2.25.0=py38h32f6830_0
   - dropbox=10.1.1=pyh9f0ad1d_0
   - esmpy=8.0.1=mpi_openmpi_py38h51f2404_0
   - fastparquet=0.4.1=py38h8790de6_0
@@ -32,7 +32,7 @@ dependencies:
   - git=2.28.0=pl526h5e3e691_1
   - gitpython=3.1.7=py_0
   - google-cloud-container=0.3.0=py38_0
-  - google-cloud-storage=1.30.0=pyh9f0ad1d_0
+  - google-cloud-storage=1.31.0=pyh9f0ad1d_0
   - holoviews=1.13.3=pyh9f0ad1d_0
   - hvplot=0.6.0=pyh9f0ad1d_0
   - h5netcdf=0.8.1=py_0
@@ -50,7 +50,7 @@ dependencies:
   - netcdf-fortran=4.5.3=mpi_openmpi_h2cce2cb_0
   - netcdf4=1.5.4=nompi_py38hec8b9af_102
   - nose=1.3.7=py38h32f6830_1004
-  - numba=0.51.0=py38hc5bc63f_0
+  - numba=0.51.1=py38hc5bc63f_0
   - openpyxl=3.0.5=py_0
   - pandas=1.1.1=py38h950e882_0
   - phantomjs=2.1.1=1 # for geoviews
@@ -60,7 +60,7 @@ dependencies:
   - pygeos=0.7.1=py38hd168ffb_0
   - pyinterp=0.3.2=mkl_py38hefd7b46_102
   - pyshp=2.1.0=py_0
-  - python=3.8.5=h1103e12_5_cpython
+  - python=3.8.5=h1103e12_7_cpython
   - python-snappy=0.5.4=py38h7cfaab3_1
   - pyyaml=5.3.1=py38h1e0a361_0
   - rasterio=1.1.5=py38h033e0f6_1
@@ -73,7 +73,7 @@ dependencies:
   - selenium=3.141.0=py38h1e0a361_1001 # for geoviews
   - shapely=1.7.1=py38hc7361b7_0
   - sparse=0.11.0=py_0
-  - statsmodels=0.11.1=py38h1e0a361_2
+  - statsmodels=0.12.0=py38h1e0a361_0
   - tini=0.18.0=h14c3975_1001
   - xarray=0.16.0=py_0
   - xarray-extras=0.4.2=py38h516909a_1000

From b086206c7e8ce0cfa5b9445c1af85accb5726f5e Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Mon, 31 Aug 2020 17:55:44 -0700
Subject: [PATCH 185/224] bump clawpack commit

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 47b6bd8..e59f389 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=e01b6b3e8c519da40b76a74e60342767b3cccab8
+    - CLAWPACK_COMMIT=70589eb6cd4dfa59ab3d8656ac09ea8ea27a8374
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From 07fbaafdbe49624eff8d3a07d877c75f2d4055f2 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Mon, 7 Sep 2020 22:21:34 +0000
Subject: [PATCH 186/224] bump clawpack commit

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e59f389..aca7b77 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=70589eb6cd4dfa59ab3d8656ac09ea8ea27a8374
+    - CLAWPACK_COMMIT=fe0722d6668ac4121a89c66dac3cf4435baa9159
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 

From 6fab4281c0692d52a4cd7461b5b3521a129e9c43 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Mon, 7 Sep 2020 22:22:34 +0000
Subject: [PATCH 187/224] rm r env from pin.py

---
 pin.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/pin.py b/pin.py
index 901a2ac..ab5e64d 100644
--- a/pin.py
+++ b/pin.py
@@ -309,12 +309,10 @@ def pin(file, dry_run):
         pin_files([SPEC_FILES[1]], dry_run=dry_run)
     elif file == 'octave':
         pin_files([SPEC_FILES[2]], dry_run=dry_run)
-    elif file == 'r':
-        pin_files([SPEC_FILES[3]], dry_run=dry_run)
     else:
         raise ValueError(
             'env type not recognized: {}'
-            'choose from "base", "notebook", "octave", "r", or "all".'
+            'choose from "base", "notebook", "octave", or "all".'
             .format(file))
 
         

From a853a8980c50e115afda0373f42ae9fc2e09e278 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Mon, 7 Sep 2020 22:23:49 +0000
Subject: [PATCH 188/224] fix elyra tmp setup.py bug

---
 notebook/Dockerfile                   |  1 +
 notebook/notebook_environment.yml     |  6 ++++--
 shared_resources/base_environment.yml | 16 ++++++++--------
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index de665ac..6d3e60b 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -80,6 +80,7 @@ RUN jupyter labextension install \
       @ryantam626/jupyterlab_code_formatter \
       dask-labextension \
       jupyterlab-plotly \
+      jupyterlab-tabular-data-editor \
       plotlywidget
 RUN jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
 
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index ac7861c..832ee0b 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -13,7 +13,7 @@ dependencies:
   - ipypublish
   - isort
 # need jupyterhub version to match that running on the hub
-  - jupyterhub=1.0.0=py38_0 # pinkeep: jupyterhub=1.0.0
+  - jupyterhub=1.0.0 # pinkeep: jupyterhub=1.0.0
   - jupyterlab
   - jupyterlab_code_formatter
   - mypy
@@ -33,4 +33,6 @@ dependencies:
   - tox
   - pip:
     - black_nbconvert
-    - elyra
+    # need to clone from repo until version > 1.1.0 to fix setup.py bug
+    # (https://github.com/elyra-ai/elyra/commit/2ad2e66ca1ba26c99d39f1d2eea21bd508ce6cdb)
+    - git+https://github.com/elyra-ai/elyra.git@2ad2e66ca1ba26c99d39f1d2eea21bd508ce6cdb#egg=elyra
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 965423f..8b21425 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -4,7 +4,7 @@ dependencies:
   - beautifulsoup4=4.9.1=py_1
   - blas=2.17=openblas
   - bottleneck=1.3.2=py38h8790de6_1
-  - boto3=1.14.51=pyh9f0ad1d_0
+  - boto3=1.14.56=pyh9f0ad1d_0
   - bqplot=0.12.16=pyh9f0ad1d_0
   - cachecontrol=0.12.6=py_0
   - cartopy=0.18.0=py38h172510d_0
@@ -19,7 +19,7 @@ dependencies:
   - descartes=1.1.0=py_4
   - distributed=2.25.0=py38h32f6830_0
   - dropbox=10.1.1=pyh9f0ad1d_0
-  - esmpy=8.0.1=mpi_openmpi_py38h51f2404_0
+  - esmpy=8.0.1=mpi_mpich_py38h08b7006_100
   - fastparquet=0.4.1=py38h8790de6_0
   - fiona=1.8.13=py38h033e0f6_1
   - fusepy=3.0.1=py_0
@@ -30,7 +30,7 @@ dependencies:
   - geotiff=1.6.0=h05acad5_0
   - geoviews=1.8.1=py_0
   - git=2.28.0=pl526h5e3e691_1
-  - gitpython=3.1.7=py_0
+  - gitpython=3.1.8=py_0
   - google-cloud-container=0.3.0=py38_0
   - google-cloud-storage=1.31.0=pyh9f0ad1d_0
   - holoviews=1.13.3=pyh9f0ad1d_0
@@ -47,17 +47,17 @@ dependencies:
   - make=4.3=h516909a_0
   - matplotlib=3.3.1=1
   - nc-time-axis=1.2.0=py_1
-  - netcdf-fortran=4.5.3=mpi_openmpi_h2cce2cb_0
+  - netcdf-fortran=4.5.3=mpi_mpich_h3923e1a_0
   - netcdf4=1.5.4=nompi_py38hec8b9af_102
   - nose=1.3.7=py38h32f6830_1004
-  - numba=0.51.1=py38hc5bc63f_0
+  - numba=0.51.2=py38hc5bc63f_0
   - openpyxl=3.0.5=py_0
   - pandas=1.1.1=py38h950e882_0
   - phantomjs=2.1.1=1 # for geoviews
   - pillow=7.2.0=py38h9776b28_1
   - pip=20.2.2=py_0
   - plotly=4.9.0=pyh9f0ad1d_0
-  - pygeos=0.7.1=py38hd168ffb_0
+  - pygeos=0.8=py38hc7361b7_0
   - pyinterp=0.3.2=mkl_py38hefd7b46_102
   - pyshp=2.1.0=py_0
   - python=3.8.5=h1103e12_7_cpython
@@ -66,13 +66,13 @@ dependencies:
   - rasterio=1.1.5=py38h033e0f6_1
   - regionmask=0.6.1=py_0
   - rtree=0.9.4=py38h08f867b_1
-  - scikit-image=0.17.2=py38hcb8c335_1
+  - scikit-image=0.17.2=py38hc5bc63f_2
   - scikit-learn=0.23.2=py38hee58b96_0
   - scipy=1.5.2=py38h8c5af15_0
   - seaborn=0.10.1=1
   - selenium=3.141.0=py38h1e0a361_1001 # for geoviews
   - shapely=1.7.1=py38hc7361b7_0
-  - sparse=0.11.0=py_0
+  - sparse=0.11.2=py_0
   - statsmodels=0.12.0=py38h1e0a361_0
   - tini=0.18.0=h14c3975_1001
   - xarray=0.16.0=py_0

From 81b7441b83716ccb962a47bbde1eadd993b4ac9c Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 13:06:44 -0700
Subject: [PATCH 189/224] drop elyra for now

---
 .travis.yml                       | 8 ++++----
 notebook/Dockerfile               | 1 +
 notebook/notebook_environment.yml | 6 ++----
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index aca7b77..db8910e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ env:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
   global:
-    - CLAWPACK_COMMIT=fe0722d6668ac4121a89c66dac3cf4435baa9159
+    - CLAWPACK_COMMIT=5f6f94c6087c520e29fbdaf823965c2d8e2ec6b2
 
 install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
@@ -25,7 +25,7 @@ install:
     $IMAGE_NAME/shared_resources"
 - "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \
     octave-worker/shared_resources && chmod -R +x octave-worker/shared_resources; fi"
-- "cd $IMAGE_NAME"
+- cd $IMAGE_NAME
 
 
 script:
@@ -45,7 +45,7 @@ deploy:
     docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH"
   on:
     all_branches: true
-    condition: $TRAVIS_BRANCH =~ ^dev
+    condition: $TRAVIS_BRANCH != master
 
 - provider: script
   script: >-
@@ -77,7 +77,7 @@ deploy:
     docker push "rhodium/octave-worker:$TRAVIS_BRANCH"
   on:
     all_branches: true
-    condition: ($TRAVIS_BRANCH =~ ^dev) && ($IMAGE_NAME == worker)
+    condition: ($TRAVIS_BRANCH != master) && ($IMAGE_NAME == worker)
 
 - provider: script
   script: >-
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 6d3e60b..ef8a41a 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -74,6 +74,7 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
 RUN jupyter labextension update --all 
 RUN jupyter labextension install \
       @bokeh/jupyter_bokeh \
+      @jupyterlab/toc \
       @jupyter-widgets/jupyterlab-manager \
       @lckr/jupyterlab_variableinspector \
       @pyviz/jupyterlab_pyviz \
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 832ee0b..50de34f 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -16,6 +16,7 @@ dependencies:
   - jupyterhub=1.0.0 # pinkeep: jupyterhub=1.0.0
   - jupyterlab
   - jupyterlab_code_formatter
+  - jupyterlab-git
   - mypy
   - nano
   - nbdime
@@ -32,7 +33,4 @@ dependencies:
   - sphinx_rtd_theme
   - tox
   - pip:
-    - black_nbconvert
-    # need to clone from repo until version > 1.1.0 to fix setup.py bug
-    # (https://github.com/elyra-ai/elyra/commit/2ad2e66ca1ba26c99d39f1d2eea21bd508ce6cdb)
-    - git+https://github.com/elyra-ai/elyra.git@2ad2e66ca1ba26c99d39f1d2eea21bd508ce6cdb#egg=elyra
+    - black_nbconvert
\ No newline at end of file

From b00f11723edc8880e651cebb1fedd2ff80e76f2e Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Mon, 7 Sep 2020 19:18:05 -0700
Subject: [PATCH 190/224] try dask-gateway

---
 notebook/notebook_environment.yml     | 3 ++-
 shared_resources/base_environment.yml | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 50de34f..e8f43de 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -13,8 +13,9 @@ dependencies:
   - ipypublish
   - isort
 # need jupyterhub version to match that running on the hub
-  - jupyterhub=1.0.0 # pinkeep: jupyterhub=1.0.0
+  - jupyterhub=1.1.0 # pinkeep: jupyterhub=1.1.0
   - jupyterlab
+  - jupyterlab-git
   - jupyterlab_code_formatter
   - jupyterlab-git
   - mypy
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 8b21425..c8a8c81 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -12,8 +12,8 @@ dependencies:
   - click=7.1.2=pyh9f0ad1d_0
   - compilers=1.1.1=0
   - dask=2.25.0=py_0
+  - dask-gateway=0.8.0=py38h32f6830_0
   - dask-glm=0.2.0=py_1
-  - dask-kubernetes=0.10.1=py_0
   - dask-ml=1.6.0=py_0
   - datashader=0.11.1=pyh9f0ad1d_0
   - descartes=1.1.0=py_4

From 0d5f9933c17a4ae5d806fae0ecb088c639cc1228 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Tue, 8 Sep 2020 09:06:41 -0700
Subject: [PATCH 191/224] drop elyra for now

---
 notebook/notebook_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index e8f43de..d7a8694 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -34,4 +34,4 @@ dependencies:
   - sphinx_rtd_theme
   - tox
   - pip:
-    - black_nbconvert
\ No newline at end of file
+    - black_nbconvert

From 37d151105b8e2688bcd5811cfd497db74b622355 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Tue, 8 Sep 2020 18:53:42 -0700
Subject: [PATCH 192/224] add lightweight scheduler image for d-g

---
 .travis.yml                                | 55 +++++++++++++-----
 notebook/Dockerfile                        |  7 ++-
 pin.py                                     | 11 ++--
 shared_resources/base_environment.yml      |  4 --
 shared_resources/scheduler_environment.yml | 11 ++++
 worker/Dockerfile                          | 58 +------------------
 worker/Dockerfile_scheduler                | 65 ++++++++++++++++++++++
 7 files changed, 132 insertions(+), 79 deletions(-)
 create mode 100644 shared_resources/scheduler_environment.yml
 create mode 100644 worker/Dockerfile_scheduler

diff --git a/.travis.yml b/.travis.yml
index db8910e..8ef86f6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,18 +9,6 @@ env:
     - CLAWPACK_COMMIT=5f6f94c6087c520e29fbdaf823965c2d8e2ec6b2
 
 install:
-- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then 
-    sed -i.bak \
-      's/image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:.*/\
-      image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_COMMIT\"'/' \
-      notebook/worker-template.yml; 
-  else
-    sed -i.bak \
-      's/image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:.*/\
-      image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_TAG\"'/' \
-      notebook/worker-template.yml; fi"
-- "rm notebook/worker-template.yml.bak"
-- "cat notebook/worker-template.yml | grep image:"
 - "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \
     $IMAGE_NAME/shared_resources"
 - "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \
@@ -29,7 +17,17 @@ install:
 
 
 script:
-- travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT .
+- "if [[ $IMAGE_NAME == notebook ]]; then
+    if [[ \"$TRAVIS_TAG\" == \"\" ]]; then
+      export WORKER_IMAGE=gcr.io\\/rhg-project-1\\/pytc-image-devbase:\"$TRAVIS_COMMIT\";
+    else
+      export WORKER_IMAGE=gcr.io\\/rhg-project-1\\/pytc-image-devbase:\"$TRAVIS_TAG\";
+    fi;
+  fi"
+- echo $WORKER_IMAGE
+- "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
+    rhodium/scheduler:local -f Dockerfile_scheduler .; fi"
+- travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT --build-arg WORKER_IMAGE .
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \
     ../octave-worker; fi"
@@ -67,6 +65,37 @@ deploy:
   on:
     tags: true
 
+# scheduler builds
+- provider: script
+  script: >-
+    docker tag rhodium/scheduler:local rhodium/scheduler:$TRAVIS_COMMIT &&
+    docker tag rhodium/scheduler:local rhodium/scheduler:$TRAVIS_BRANCH &&
+    docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
+    docker push "rhodium/scheduler:$TRAVIS_COMMIT" &&
+    docker push "rhodium/scheduler:$TRAVIS_BRANCH"
+  on:
+    all_branches: true
+    condition: ($TRAVIS_BRANCH != master) && ($IMAGE_NAME == worker)
+
+- provider: script
+  script: >-
+    docker tag rhodium/scheduler:local rhodium/scheduler:latest &&
+    docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
+    docker push "rhodium/scheduler:$TRAVIS_COMMIT" &&
+    docker push "rhodium/scheduler:latest"
+  on:
+    branch: master
+    condition: $IMAGE_NAME = worker
+
+- provider: script
+  script: >-
+    docker tag rhodium/scheduler:local rhodium/scheduler:$TRAVIS_TAG &&
+    docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" &&
+    docker push "rhodium/scheduler:$TRAVIS_TAG"
+  on:
+    tags: true
+    condition: $IMAGE_NAME = worker
+
 # octave-worker builds
 - provider: script
   script: >-
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index ef8a41a..45b7785 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -44,6 +44,7 @@ RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recomme
 # set up conda channels
 RUN mkdir /opt/conda/specs
 COPY shared_resources/base_environment.yml /opt/conda/specs
+COPY shared_resources/scheduler_environment.yml /opt/conda/specs
 COPY notebook_environment.yml /opt/conda/specs
 RUN conda config --add channels conda-forge && \
   conda config --set channel_priority strict
@@ -52,7 +53,8 @@ RUN conda config --add channels conda-forge && \
 RUN conda update -n base conda
 
 #  update environemnt with common packages across worker and nb
-RUN conda env update -f /opt/conda/specs/base_environment.yml
+RUN conda env update -f /opt/conda/specs/scheduler_environment.yml && \
+  conda env update -f /opt/conda/specs/base_environment.yml
 
 # need access to settings/page_config.json for elyra
 USER root
@@ -133,6 +135,9 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
 RUN sudo rm -rf /var/lib/apt/lists/* /tempdir
 RUN conda clean -yaf
 
+## configure correct default image for dask
+ENV DASK_GATEWAY__CLUSTER__OPTIONS__IMAGE ${WORKER_IMAGE}
+
 ## prepare container
 WORKDIR $HOME
 ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
diff --git a/pin.py b/pin.py
index ab5e64d..68dc324 100644
--- a/pin.py
+++ b/pin.py
@@ -27,8 +27,9 @@
 
 SPEC_FILES = [
     ('shared_resources/base_environment.yml', 'base'),
+    ('shared_resources/scheduler_environment.yml', 'base'),
     ('notebook/notebook_environment.yml', 'base'),
-    ('octave-worker/octave_environment.yml', 'base')
+    ('octave-worker/octave_environment.yml', 'base'),
 ]
 
 
@@ -332,16 +333,16 @@ def unpin(file, dry_run):
         unpin_files(SPEC_FILES, dry_run=dry_run)
     elif file == 'base':
         unpin_files([SPEC_FILES[0]], dry_run=dry_run)
-    elif file == 'notebook':
+    elif file == 'scheduler':
         unpin_files([SPEC_FILES[1]], dry_run=dry_run)
-    elif file == 'octave':
+    elif file == 'notebook':
         unpin_files([SPEC_FILES[2]], dry_run=dry_run)
-    elif file == 'r':
+    elif file == 'octave':
         unpin_files([SPEC_FILES[3]], dry_run=dry_run)
     else:
         raise ValueError(
             'env type not recognized: {}'
-            'choose from "base", "notebook", "octave", "r", or "all".'
+            'choose from "base", "scheduler", "notebook", "octave", or "all".'
             .format(file))
 
 
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index c8a8c81..dea1b07 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -11,13 +11,10 @@ dependencies:
   - cftime=1.2.1=py38h8790de6_0
   - click=7.1.2=pyh9f0ad1d_0
   - compilers=1.1.1=0
-  - dask=2.25.0=py_0
-  - dask-gateway=0.8.0=py38h32f6830_0
   - dask-glm=0.2.0=py_1
   - dask-ml=1.6.0=py_0
   - datashader=0.11.1=pyh9f0ad1d_0
   - descartes=1.1.0=py_4
-  - distributed=2.25.0=py38h32f6830_0
   - dropbox=10.1.1=pyh9f0ad1d_0
   - esmpy=8.0.1=mpi_mpich_py38h08b7006_100
   - fastparquet=0.4.1=py38h8790de6_0
@@ -52,7 +49,6 @@ dependencies:
   - nose=1.3.7=py38h32f6830_1004
   - numba=0.51.2=py38hc5bc63f_0
   - openpyxl=3.0.5=py_0
-  - pandas=1.1.1=py38h950e882_0
   - phantomjs=2.1.1=1 # for geoviews
   - pillow=7.2.0=py38h9776b28_1
   - pip=20.2.2=py_0
diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
new file mode 100644
index 0000000..a74ed7e
--- /dev/null
+++ b/shared_resources/scheduler_environment.yml
@@ -0,0 +1,11 @@
+channels:
+    - conda-forge
+dependencies:
+    - aiohttp=3.6.2=py38h516909a_0
+    - dask=2.25.0=py_0
+    - distributed=2.25.0=py38h32f6830_0
+    - dask-gateway=0.8.0=py38h32f6830_0
+    - numpy=1.19.1=py38hbc27379_2
+    - pandas=1.1.1=py38h950e882_0
+name: base
+  
\ No newline at end of file
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 49a527b..0390f52 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -1,52 +1,7 @@
 ## using same base image as the "base image for the base image" of the notebook image
-FROM ubuntu:focal-20200703@sha256:d5a6519d9f048100123c568eb83f7ef5bfcad69b01424f420f17c932b00dea76
+FROM rhodium/scheduler:local
 ARG DEBIAN_FRONTEND=noninteractive
 
-# needed to properly install packages that use numpy libraries
-ENV NPY_DISTUTILS_APPEND_FLAGS=1
-
-# needed so that matplotlib will work headless
-ENV MPLBACKEND=Agg
-
-## filepath curation
-COPY shared_resources /tempdir
-COPY add_service_creds.py /usr/bin
-COPY prepare.sh /usr/bin
-
-## perform a bunch of common actions
-RUN bash /tempdir/common.sh
-
-###########
-## install miniconda (following jupyter/base-notebook
-RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
-    locale-gen
-    
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-ENV CONDA_DIR=/opt/conda \
-    SHELL=/bin/bash \
-    LC_ALL=en_US.UTF-8 \
-    LANG=en_US.UTF-8 \
-    LANGUAGE=en_US.UTF-8
-ENV PATH=$CONDA_DIR/bin:$PATH
-
-ENV MINICONDA_VERSION=4.8.3 \
-    MINICONDA_MD5=d63adf39f2c220950a063e0529d4ff74 \
-    CONDA_VERSION=4.8.3
-
-WORKDIR /tmp
-RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
-    echo "${MINICONDA_MD5} *Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
-    /bin/bash Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
-    rm Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
-    conda config --system --prepend channels conda-forge && \
-    conda config --system --set channel_priority strict && \
-    conda update -n base --yes conda && \
-    conda update --all --yes
-###########
-
-## set up python env
-RUN mkdir /opt/conda/specs
-COPY shared_resources/base_environment.yml /opt/conda/specs
 RUN conda env update -f /opt/conda/specs/base_environment.yml
 RUN conda list -n base
 
@@ -86,13 +41,4 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
 ## clean up
 RUN rm -rf /var/lib/apt/lists/* /tempdir \
     && conda clean -yaf \
-    && sudo apt-get clean
-
-## prepare container
-ENV OMP_NUM_THREADS=1
-ENV MKL_NUM_THREADS=1
-ENV OPENBLAS_NUM_THREADS=1
-
-WORKDIR /
-
-ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
+    && sudo apt-get clean
\ No newline at end of file
diff --git a/worker/Dockerfile_scheduler b/worker/Dockerfile_scheduler
new file mode 100644
index 0000000..0c29a06
--- /dev/null
+++ b/worker/Dockerfile_scheduler
@@ -0,0 +1,65 @@
+## using same base image as the "base image for the base image" of the notebook image
+FROM ubuntu:focal-20200703@sha256:d5a6519d9f048100123c568eb83f7ef5bfcad69b01424f420f17c932b00dea76
+ARG DEBIAN_FRONTEND=noninteractive
+
+# needed to properly install packages that use numpy libraries
+ENV NPY_DISTUTILS_APPEND_FLAGS=1
+
+# needed so that matplotlib will work headless
+ENV MPLBACKEND=Agg
+
+## filepath curation
+COPY shared_resources /tempdir
+COPY add_service_creds.py /usr/bin
+COPY prepare.sh /usr/bin
+
+## perform a bunch of common actions
+RUN bash /tempdir/common.sh
+
+###########
+## install miniconda (following jupyter/base-notebook
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    locale-gen
+    
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV CONDA_DIR=/opt/conda \
+    SHELL=/bin/bash \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
+ENV PATH=$CONDA_DIR/bin:$PATH
+
+ENV MINICONDA_VERSION=4.8.3 \
+    MINICONDA_MD5=d63adf39f2c220950a063e0529d4ff74 \
+    CONDA_VERSION=4.8.3
+
+WORKDIR /tmp
+RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    echo "${MINICONDA_MD5} *Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
+    /bin/bash Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    conda config --system --prepend channels conda-forge && \
+    conda config --system --set channel_priority strict && \
+    conda update -n base --yes conda && \
+    conda update --all --yes
+###########
+
+## set up python env
+RUN mkdir /opt/conda/specs
+COPY shared_resources/scheduler_environment.yml /opt/conda/specs
+COPY shared_resources/base_environment.yml /opt/conda/specs
+RUN conda env update -f /opt/conda/specs/scheduler_environment.yml
+
+## clean up
+RUN rm -rf /var/lib/apt/lists/* /tmp/*\
+    && conda clean -yaf \
+    && sudo apt-get clean
+
+## prepare container
+ENV OMP_NUM_THREADS=1
+ENV MKL_NUM_THREADS=1
+ENV OPENBLAS_NUM_THREADS=1
+
+WORKDIR /
+
+ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
\ No newline at end of file

From 518c8e8814095d1016827578ae19ad2ccf0c2f30 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Tue, 8 Sep 2020 19:36:22 -0700
Subject: [PATCH 193/224] set scheduler image too

---
 .travis.yml                                | 20 +++++++++++---------
 notebook/Dockerfile                        |  3 ++-
 shared_resources/scheduler_environment.yml |  2 ++
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 8ef86f6..a8eefbf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,23 +11,25 @@ env:
 install:
 - "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \
     $IMAGE_NAME/shared_resources"
+- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then
+    export WORKER_IMAGE=gcr.io/rhg-project-1/pytc-image-gateway:latest;
+    export SCHEDULER_IMAGE=rhodium/scheduler:\"$TRAVIS_COMMIT\";
+  else
+    export WORKER_IMAGE=gcr.io/rhg-project-1/pytc-image-gateway:latest;
+    export SCHEDULER_IMAGE=rhodium/scheduler:\"$TRAVIS_TAG\";
+  fi;"
+- echo $WORKER_IMAGE
+- echo $SCHEULER_IMAGE
 - "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \
     octave-worker/shared_resources && chmod -R +x octave-worker/shared_resources; fi"
 - cd $IMAGE_NAME
 
 
 script:
-- "if [[ $IMAGE_NAME == notebook ]]; then
-    if [[ \"$TRAVIS_TAG\" == \"\" ]]; then
-      export WORKER_IMAGE=gcr.io\\/rhg-project-1\\/pytc-image-devbase:\"$TRAVIS_COMMIT\";
-    else
-      export WORKER_IMAGE=gcr.io\\/rhg-project-1\\/pytc-image-devbase:\"$TRAVIS_TAG\";
-    fi;
-  fi"
-- echo $WORKER_IMAGE
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/scheduler:local -f Dockerfile_scheduler .; fi"
-- travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT --build-arg CLAWPACK_COMMIT --build-arg WORKER_IMAGE .
+- "travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT \
+    --build-arg CLAWPACK_COMMIT --build-arg WORKER_IMAGE --build-arg SCHEDULER_IMAGE ."
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \
     ../octave-worker; fi"
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 45b7785..46fec83 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -136,7 +136,8 @@ RUN sudo rm -rf /var/lib/apt/lists/* /tempdir
 RUN conda clean -yaf
 
 ## configure correct default image for dask
-ENV DASK_GATEWAY__CLUSTER__OPTIONS__IMAGE ${WORKER_IMAGE}
+ENV DASK_GATEWAY__CLUSTER__OPTIONS__WORKER_IMAGE ${WORKER_IMAGE}
+ENV DASK_GATEWAY__CLUSTER__OPTIONS__SCHEDULER_IMAGE ${SCHEDULER_IMAGE}
 
 ## prepare container
 WORKDIR $HOME
diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
index a74ed7e..b8a7434 100644
--- a/shared_resources/scheduler_environment.yml
+++ b/shared_resources/scheduler_environment.yml
@@ -1,3 +1,5 @@
+# Only the packages necessary to build a dask-gateway scheduler. As seen here:
+# https://github.com/dask/dask-gateway/blob/master/dask-gateway/Dockerfile
 channels:
     - conda-forge
 dependencies:

From 53441912489a74c46b71f8ec9432a78542cb18b8 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Tue, 8 Sep 2020 19:58:34 -0700
Subject: [PATCH 194/224] move tini to scheduler image

---
 .travis.yml                                | 2 +-
 shared_resources/base_environment.yml      | 1 -
 shared_resources/scheduler_environment.yml | 1 +
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a8eefbf..0b17cdd 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,7 +19,7 @@ install:
     export SCHEDULER_IMAGE=rhodium/scheduler:\"$TRAVIS_TAG\";
   fi;"
 - echo $WORKER_IMAGE
-- echo $SCHEULER_IMAGE
+- echo $SCHEDULER_IMAGE
 - "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \
     octave-worker/shared_resources && chmod -R +x octave-worker/shared_resources; fi"
 - cd $IMAGE_NAME
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index dea1b07..8c548b3 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -70,7 +70,6 @@ dependencies:
   - shapely=1.7.1=py38hc7361b7_0
   - sparse=0.11.2=py_0
   - statsmodels=0.12.0=py38h1e0a361_0
-  - tini=0.18.0=h14c3975_1001
   - xarray=0.16.0=py_0
   - xarray-extras=0.4.2=py38h516909a_1000
   - xesmf=0.3.0=py_0
diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
index b8a7434..7fa5312 100644
--- a/shared_resources/scheduler_environment.yml
+++ b/shared_resources/scheduler_environment.yml
@@ -9,5 +9,6 @@ dependencies:
     - dask-gateway=0.8.0=py38h32f6830_0
     - numpy=1.19.1=py38hbc27379_2
     - pandas=1.1.1=py38h950e882_0
+    - tini=0.18.0=h14c3975_1001
 name: base
   
\ No newline at end of file

From 921116521e24c9edb2061bc6e5221aea22ec1fc8 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Tue, 8 Sep 2020 21:13:23 -0700
Subject: [PATCH 195/224] get build args in there

---
 notebook/Dockerfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 46fec83..0098bef 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,6 +1,11 @@
 FROM jupyter/base-notebook@sha256:6e276c402aa19799798b84b46e0e08fc03be193de9a81490158ca4d9026b8cae
 ARG DEBIAN_FRONTEND=noninteractive
 
+# pull in build args
+ARG CLAWPACK_COMMIT
+ARG WORKER_IMAGE
+ARG SCHEDULER_IMAGE
+
 ## needed to make sure things with cython compile correctly
 ## (this will eventually become default in numpy)
 ENV NPY_DISTUTILS_APPEND_FLAGS=1
@@ -98,7 +103,6 @@ COPY nbdime_config.json /opt/conda/etc/jupyter
 ## clawpack added for coastal-specific image
 ##################
 
-ARG CLAWPACK_COMMIT
 ENV CLAW=/clawpack
 ENV NETCDF4_DIR=/opt/conda
 ENV FC=gfortran

From f189a4b39cbf99d349f2c1546ca648909dc8285a Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 06:23:06 -0700
Subject: [PATCH 196/224] add dask-kube back for compatibility

---
 shared_resources/base_environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 8c548b3..3ac3b37 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -12,6 +12,7 @@ dependencies:
   - click=7.1.2=pyh9f0ad1d_0
   - compilers=1.1.1=0
   - dask-glm=0.2.0=py_1
+  - dask-kubernetes=0.10.1=py_0
   - dask-ml=1.6.0=py_0
   - datashader=0.11.1=pyh9f0ad1d_0
   - descartes=1.1.0=py_4

From 0ec9d67fb2cce7f8461b38e3e2a6eb2a35ab883c Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 06:25:09 -0700
Subject: [PATCH 197/224] add worker template back for compat

---
 .travis.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 0b17cdd..d164f62 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,10 +14,20 @@ install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then
     export WORKER_IMAGE=gcr.io/rhg-project-1/pytc-image-gateway:latest;
     export SCHEDULER_IMAGE=rhodium/scheduler:\"$TRAVIS_COMMIT\";
+    sed -i.bak \
+      's/image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:.*/\
+      image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_COMMIT\"'/' \
+      notebook/worker-template.yml; 
   else
     export WORKER_IMAGE=gcr.io/rhg-project-1/pytc-image-gateway:latest;
     export SCHEDULER_IMAGE=rhodium/scheduler:\"$TRAVIS_TAG\";
+    sed -i.bak \
+      's/image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:.*/\
+      image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_TAG\"'/' \
+      notebook/worker-template.yml;
   fi;"
+- rm notebook/worker-template.yml.bak
+- "cat notebook/worker-template.yml | grep image:"
 - echo $WORKER_IMAGE
 - echo $SCHEDULER_IMAGE
 - "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \

From e79168660c9c856ca644b260417faa5a6d3ab780 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 08:10:50 -0700
Subject: [PATCH 198/224] fix gcsfuse auth and drop worker-template

---
 .travis.yml                           |  10 --
 bump.py                               |   1 -
 notebook/Dockerfile                   |   5 +-
 notebook/add_service_creds.py         |  38 ------
 notebook/config.yaml                  |  12 +-
 notebook/prepare.sh                   |  20 ++-
 notebook/run_sql_proxy.py             | 168 --------------------------
 notebook/set_gateway_opts.py          |  36 ++++++
 notebook/worker-template.yml          |  30 -----
 shared_resources/base_environment.yml |   1 -
 10 files changed, 48 insertions(+), 273 deletions(-)
 delete mode 100755 notebook/add_service_creds.py
 delete mode 100755 notebook/run_sql_proxy.py
 create mode 100644 notebook/set_gateway_opts.py
 delete mode 100755 notebook/worker-template.yml

diff --git a/.travis.yml b/.travis.yml
index d164f62..0b17cdd 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,20 +14,10 @@ install:
 - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then
     export WORKER_IMAGE=gcr.io/rhg-project-1/pytc-image-gateway:latest;
     export SCHEDULER_IMAGE=rhodium/scheduler:\"$TRAVIS_COMMIT\";
-    sed -i.bak \
-      's/image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:.*/\
-      image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_COMMIT\"'/' \
-      notebook/worker-template.yml; 
   else
     export WORKER_IMAGE=gcr.io/rhg-project-1/pytc-image-gateway:latest;
     export SCHEDULER_IMAGE=rhodium/scheduler:\"$TRAVIS_TAG\";
-    sed -i.bak \
-      's/image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:.*/\
-      image: gcr.io\\/rhg-project-1\\/pytc-image-devbase:'\"$TRAVIS_TAG\"'/' \
-      notebook/worker-template.yml;
   fi;"
-- rm notebook/worker-template.yml.bak
-- "cat notebook/worker-template.yml | grep image:"
 - echo $WORKER_IMAGE
 - echo $SCHEDULER_IMAGE
 - "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \
diff --git a/bump.py b/bump.py
index 42297a5..5a14384 100644
--- a/bump.py
+++ b/bump.py
@@ -5,7 +5,6 @@
 
 SEARCH_PATTERNS = [
     ('.travis.yml', r'(?P<pre>.*TAG=)(?P<date>\d{4}-\d{2}-\d{2})\.?(?P<ver>\d{2})?(?P<post>[.\s]*)$'),
-    ('notebook/worker-template.yml', r'(?P<pre>.*image: rhodium/worker:)(?P<date>\d{4}-\d{2}-\d{2})\.?(?P<ver>\d{2})?(?P<post>[.\s]*)$'),
     ('jupyter-config.yml', r'(?P<pre>.*tag: )(?P<date>\d{4}-\d{2}-\d{2})\.?(?P<ver>\d{2})?(?P<post>[.\s]*)$')]
 
 
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 0098bef..cd36d3b 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -23,10 +23,7 @@ USER $NB_USER
 
 ## filepath curation
 RUN sudo mkdir /pre-home && sudo chown -R $NB_USER /pre-home
-COPY worker-template.yml /pre-home
-COPY add_service_creds.py /pre-home
-COPY run_sql_proxy.py /pre-home
-COPY config.yaml /pre-home
+COPY config.yaml set_gateway_opts.py /pre-home
 
 COPY shared_resources /tempdir
 
diff --git a/notebook/add_service_creds.py b/notebook/add_service_creds.py
deleted file mode 100755
index 44163e1..0000000
--- a/notebook/add_service_creds.py
+++ /dev/null
@@ -1,38 +0,0 @@
-import json, yaml, os, glob
-
-
-def add_service_creds():
-
-    with open('/home/jovyan/worker-template.yml', 'r') as f:
-        WORKER_TEMPLATE = yaml.safe_load(f)
-        
-    env_vars = []
-    creds = {}
-
-    for env in WORKER_TEMPLATE['spec']['containers'][0]['env']:
-        if 'GCSFUSE_TOKEN' in env['name']:
-            continue
-        elif 'GCSFUSE_TOKENS' in env['name']:
-            creds.update(env['value'])
-        else:
-            env_vars.append(env)
-
-    for f in glob.glob('/home/jovyan/service-account-credentials/*.json'):
-        bucket = os.path.splitext(os.path.basename(f))[0]
-
-        with open(f, 'r') as f:
-            creds[bucket] = json.load(f)
-
-    env_vars.append(
-        {'name': 'GCSFUSE_TOKENS', 'value': json.dumps(creds)})
-    
-    WORKER_TEMPLATE['spec']['containers'][0]['env'] = env_vars
-
-    with open('/home/jovyan/worker-template.yml', 'w') as f:
-        f.write(yaml.dump(WORKER_TEMPLATE))
-
-    print('worker-template.yml updated')
-
-
-if __name__ == '__main__':
-    add_service_creds()
\ No newline at end of file
diff --git a/notebook/config.yaml b/notebook/config.yaml
index 3e8ec1a..34dee2a 100755
--- a/notebook/config.yaml
+++ b/notebook/config.yaml
@@ -17,16 +17,10 @@ logging:
   tornado.application: error
 
 # the below section gives defaults for launching clusters from the dask
-# labextension viewer. We could customize this but right now it should load
-# a default cluster from the worker-template.yml file (as sepecified in the
-# following section)
+# labextension viewer. We could customize this further
 labextension:
   factory:
-    module: dask_kubernetes
-    class: KubeCluster
+    module: dask-gateway
+    class: GatewayCluster
     args: []
     kwargs: {}
-
-kubernetes:
-  worker-template-path: "/home/{NB_USER}/worker-template.yml"
-  name: "dask-{JUPYTERHUB_USER}-{uuid}"
diff --git a/notebook/prepare.sh b/notebook/prepare.sh
index 6f7b95e..6b38c04 100755
--- a/notebook/prepare.sh
+++ b/notebook/prepare.sh
@@ -2,16 +2,14 @@
 
 set -x
 
-echo "Copy Dask configuration files from pre-load directory into home/.config"
-mkdir -p /home/jovyan/.config/dask
-cp --update -r -v /pre-home/config.yaml /home/jovyan/.config/dask/
+echo "Copy Dask configuration files from pre-load directory into opt/conda/etc/dask/"
+mkdir -p /opt/conda/etc/dask
+cp --update -r -v /pre-home/config.yaml /opt/conda/etc/dask/
 
-# should probably pick one of these!!! The second is new, but is implied by the
-# cp /pre-home below, and we actually only read the version in ~ in rhg_compute_tools.
-cp -r -v /pre-home/worker-template.yml /home/jovyan/.config/dask/
-cp -r -v /pre-home/worker-template.yml /home/jovyan/
+# set credentials for use when starting workers with dask-gateway
+python /pre-home/set_gateway_opts.py
 
-sudo rm /pre-home/config.yaml
+sudo rm /pre-home/config.yaml /pre-home/set_gateway_opts.py
 
 echo "Copy files from pre-load directory into home"
 cp --update -r -v /pre-home/. /home/jovyan
@@ -30,14 +28,12 @@ do
             echo "Mounting $bucket to /gcs/${bucket}";
             mkdir -p "/gcs/$bucket";
             /usr/bin/gcsfuse --key-file="$f" "$bucket" "/gcs/${bucket}";
+            echo "Including $bucket in dask-gateway options";
         fi;
     fi;
 done
 
-if [ -f "/home/jovyan/worker-template.yml" ]; then
-    echo "appending service-account-credentials to worker-template";
-    python /home/jovyan/add_service_creds.py;
-fi
+
 
 # needed for CLAWPACK to not throw segfaults sometimes
 ulimit -s unlimited
diff --git a/notebook/run_sql_proxy.py b/notebook/run_sql_proxy.py
deleted file mode 100755
index 7450e01..0000000
--- a/notebook/run_sql_proxy.py
+++ /dev/null
@@ -1,168 +0,0 @@
-'''
-Run an SQL proxy server to enable connections to a cloud sql instance
-
-To use, create a file at /home/jovyan/setup.cfg with the following contents:
-
-.. code-block:: bash
-
-    [sql-proxy]
-
-    SQL_INSTANCE = {project}:{region}:{instance}=tcp:{port}
-    SQL_TOKEN_FILE = /path/to/credentials-file.json
-
-modifying the `SQL_INSTANCE` and `SQL_TOKEN` values to match your server's
-configuration.
-
-Then, run `python run_sql_proxy.py`. This will start an SQL proxy and will also
-add these credentials to your ~/worker-template.yml file.
-
-When the process is killed (through `^C` or by killing the process) the worker
-template will be returned to its previous state.
-
-'''
-
-import os
-import json
-import yaml
-import configparser
-import subprocess
-import signal
-import time
-
-
-def get_sql_service_account_token(sql_token_file):
-    if sql_token_file is None:
-        return
-
-    try:
-        with open(sql_token_file, 'r') as f:
-            return json.load(f)
-
-    except (OSError, IOError):
-        return
-
-
-class add_sql_proxy_to_worker_spec(object):
-    kill_now = False
-
-    def __init__(self, sql_instance, sql_token):
-        self.original_worker_template = None
-        self.sql_instance = sql_instance
-        self.sql_token = sql_token
-
-        self.sql_proxy_process = None
-        
-        # handle sigint
-        signal.signal(signal.SIGINT, self.return_worker_spec_to_original_state)
-        signal.signal(signal.SIGTERM, self.return_worker_spec_to_original_state)
-
-    def __enter__(self):
-        sql_instance = self.sql_instance
-        sql_token = self.sql_token
-
-        if (sql_instance is None) or (sql_token is None):
-            return
-
-        try:
-            with open('/home/jovyan/worker-template.yml', 'r') as f:
-                self.original_worker_template = f.read()
-                worker_template_modified = yaml.safe_load(self.original_worker_template)
-        
-        except (OSError, IOError, ValueError):
-            return
-            
-        env_vars = []
-
-        for env in worker_template_modified['spec']['containers'][0]['env']:
-            if 'SQL_INSTANCE' in env['name']:
-                continue
-            elif 'SQL_TOKEN' in env['name']:
-                continue
-            else:
-                env_vars.append(env)
-
-        env_vars.append(
-            {'name': 'SQL_TOKEN', 'value': json.dumps(sql_token)})
-
-        env_vars.append(
-            {'name': 'SQL_INSTANCE', 'value': sql_instance})
-        
-        worker_template_modified['spec']['containers'][0]['env'] = env_vars
-
-        with open('/home/jovyan/worker-template.yml', 'w') as f:
-            f.write(yaml.dump(worker_template_modified))
-
-        print('proxy added to worker-template.yml')
-
-
-    def maybe_start_sql_proxy(self, sql_instance, sql_token_file):
-        if (sql_instance is None) or (sql_token_file is None):
-            return
-        
-        p = subprocess.Popen([
-            '/usr/bin/cloud_sql_proxy',
-            '-instances',
-            sql_instance,
-            '-credential_file',
-            sql_token_file])
-
-        self.sql_proxy_process = p
-
-        p.wait()
-
-
-    def return_worker_spec_to_original_state(self, *args):
-        if self.original_worker_template is None:
-            return
-
-        with open('/home/jovyan/worker-template.yml', 'w') as f:
-            f.write(self.original_worker_template)        
-
-        print('proxy removed from worker-template.yml')
-
-        if (
-                (self.sql_proxy_process is not None)
-                and (self.sql_proxy_process.poll() is None)):
-
-            try:
-                self.sql_proxy_process.kill()
-            except:
-                pass
-        
-        self.kill_now = True
-
-
-    def __exit__(self, *errs):
-        self.return_worker_spec_to_original_state()
-
-
-def handle_sql_config():
-    config = configparser.ConfigParser()
-
-    if not os.path.isfile('/home/jovyan/setup.cfg'):
-        return
-
-    config.read('/home/jovyan/setup.cfg')
-    
-    if not 'sql-proxy' in config.sections():
-        return
-
-    sql_instance = config['sql-proxy'].get('SQL_INSTANCE')
-    sql_token_file = config['sql-proxy'].get('SQL_TOKEN_FILE')
-    sql_token = get_sql_service_account_token(sql_token_file)
-
-    sql_proxy = add_sql_proxy_to_worker_spec(sql_instance, sql_token)
-    
-    with sql_proxy:
-        sql_proxy.maybe_start_sql_proxy(sql_instance, sql_token_file)
-
-    # wait for sql_proxy to exit gracefully
-    while True:
-        if sql_proxy.kill_now:
-            break
-
-        time.sleep(1)
-
-
-if __name__ == "__main__":
-    handle_sql_config()
diff --git a/notebook/set_gateway_opts.py b/notebook/set_gateway_opts.py
new file mode 100644
index 0000000..950bdaa
--- /dev/null
+++ b/notebook/set_gateway_opts.py
@@ -0,0 +1,36 @@
+from pathlib import Path
+import json
+import yaml
+import os
+
+cred_files = Path("/home/jovyan/service-account-credentials").glob("*.json")
+
+out_file = Path("/opt/conda/etc/dask/gateway.yaml")
+out_file.parent.mkdir(exist_ok=True, parents=True)
+
+# get tokens
+tokens = {}
+for fpath in cred_files:
+    bucket = fpath.stem
+    with open(fpath, "r") as file:
+        tokens[bucket] = json.load(file)
+
+# get image names
+scheduler_image = os.environ["JUPYTER_IMAGE_SPEC"].replace("/notebook:","/scheduler:")
+worker_image = os.environ["JUPYTER_IMAGE_SPEC"].replace("/notebook:","/worker:")
+
+# create config dict
+config = {
+    "gateway": {
+        "cluster": {
+            "options": {
+                "gcsfuse_tokens": str(tokens),
+                "scheduler_image": scheduler_image,
+                "worker_image": worker_image
+            }
+        }
+    }
+}
+
+with open(out_file, "w") as fout:
+    yaml.safe_dump(config, fout)
\ No newline at end of file
diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml
deleted file mode 100755
index 2e6915b..0000000
--- a/notebook/worker-template.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-metadata:
-spec:
-  restartPolicy: Never
-  containers:
-  - args:
-    - dask-worker
-    - --nthreads
-    - '1'
-    - --no-dashboard
-    - --memory-limit
-    - 11.5GB
-    - --death-timeout
-    - '60'
-    env:
-      - name: GCSFUSE_BUCKET
-        value: rhg-data
-    image: gcr.io/rhg-project-1/pytc-image-devbase:WILL_BE_REPLACED_BY_TRAVIS
-    imagePullPolicy: Always
-    name: dask-worker
-    securityContext:
-      capabilities:
-        add: [SYS_ADMIN]
-      privileged: true
-    resources:
-      limits:
-        cpu: "1.75"
-        memory: 11.5G
-      requests:
-        cpu: "1.75"
-        memory: 11.5G
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 3ac3b37..8c548b3 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -12,7 +12,6 @@ dependencies:
   - click=7.1.2=pyh9f0ad1d_0
   - compilers=1.1.1=0
   - dask-glm=0.2.0=py_1
-  - dask-kubernetes=0.10.1=py_0
   - dask-ml=1.6.0=py_0
   - datashader=0.11.1=pyh9f0ad1d_0
   - descartes=1.1.0=py_4

From 2c0df18ba8207c450c000b8452f1b951fbaaecc8 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 08:42:49 -0700
Subject: [PATCH 199/224] fix setting of gateway opts

---
 .travis.yml                  | 11 +----------
 notebook/Dockerfile          |  6 ------
 notebook/set_gateway_opts.py |  2 +-
 3 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0b17cdd..37de3ca 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,15 +11,6 @@ env:
 install:
 - "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \
     $IMAGE_NAME/shared_resources"
-- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then
-    export WORKER_IMAGE=gcr.io/rhg-project-1/pytc-image-gateway:latest;
-    export SCHEDULER_IMAGE=rhodium/scheduler:\"$TRAVIS_COMMIT\";
-  else
-    export WORKER_IMAGE=gcr.io/rhg-project-1/pytc-image-gateway:latest;
-    export SCHEDULER_IMAGE=rhodium/scheduler:\"$TRAVIS_TAG\";
-  fi;"
-- echo $WORKER_IMAGE
-- echo $SCHEDULER_IMAGE
 - "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \
     octave-worker/shared_resources && chmod -R +x octave-worker/shared_resources; fi"
 - cd $IMAGE_NAME
@@ -29,7 +20,7 @@ script:
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/scheduler:local -f Dockerfile_scheduler .; fi"
 - "travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT \
-    --build-arg CLAWPACK_COMMIT --build-arg WORKER_IMAGE --build-arg SCHEDULER_IMAGE ."
+    --build-arg CLAWPACK_COMMIT ."
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \
     ../octave-worker; fi"
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index cd36d3b..b2b5bcb 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -3,8 +3,6 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 # pull in build args
 ARG CLAWPACK_COMMIT
-ARG WORKER_IMAGE
-ARG SCHEDULER_IMAGE
 
 ## needed to make sure things with cython compile correctly
 ## (this will eventually become default in numpy)
@@ -136,10 +134,6 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
 RUN sudo rm -rf /var/lib/apt/lists/* /tempdir
 RUN conda clean -yaf
 
-## configure correct default image for dask
-ENV DASK_GATEWAY__CLUSTER__OPTIONS__WORKER_IMAGE ${WORKER_IMAGE}
-ENV DASK_GATEWAY__CLUSTER__OPTIONS__SCHEDULER_IMAGE ${SCHEDULER_IMAGE}
-
 ## prepare container
 WORKDIR $HOME
 ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
diff --git a/notebook/set_gateway_opts.py b/notebook/set_gateway_opts.py
index 950bdaa..d48a7d7 100644
--- a/notebook/set_gateway_opts.py
+++ b/notebook/set_gateway_opts.py
@@ -24,7 +24,7 @@
     "gateway": {
         "cluster": {
             "options": {
-                "gcsfuse_tokens": str(tokens),
+                "gcsfuse_tokens": json.dumps(tokens).replace("{","{{").replace("}","}}"),
                 "scheduler_image": scheduler_image,
                 "worker_image": worker_image
             }

From f75156d67d8d395a416c893913984b8a9665ab40 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 08:58:29 -0700
Subject: [PATCH 200/224] fix copy in notebook/dockerfile

---
 notebook/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index b2b5bcb..01d632b 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -21,7 +21,7 @@ USER $NB_USER
 
 ## filepath curation
 RUN sudo mkdir /pre-home && sudo chown -R $NB_USER /pre-home
-COPY config.yaml set_gateway_opts.py /pre-home
+COPY config.yaml set_gateway_opts.py /pre-home/
 
 COPY shared_resources /tempdir
 

From 9d232ef81e9a19980b93a3d7adb9d1cc3e004537 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 11:02:55 -0700
Subject: [PATCH 201/224] move lz4 to scheduler

---
 shared_resources/base_environment.yml      | 1 -
 shared_resources/scheduler_environment.yml | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 8c548b3..1402fbd 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -40,7 +40,6 @@ dependencies:
   - kubernetes=1.18.8=0
   - libmatio=1.5.17=hf54d351_0
   - lxml=4.5.2=py38hbb43d70_0
-  - lz4=3.1.0=py38h66f7c9e_0
   - make=4.3=h516909a_0
   - matplotlib=3.3.1=1
   - nc-time-axis=1.2.0=py_1
diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
index 7fa5312..cc60ed2 100644
--- a/shared_resources/scheduler_environment.yml
+++ b/shared_resources/scheduler_environment.yml
@@ -7,6 +7,7 @@ dependencies:
     - dask=2.25.0=py_0
     - distributed=2.25.0=py38h32f6830_0
     - dask-gateway=0.8.0=py38h32f6830_0
+    - lz4=3.1.0=py38h66f7c9e_0
     - numpy=1.19.1=py38hbc27379_2
     - pandas=1.1.1=py38h950e882_0
     - tini=0.18.0=h14c3975_1001

From f5898c601c81bd14fb6ffaee1f561555a64bd5f9 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 16:58:06 -0700
Subject: [PATCH 202/224] update rhg_compute_tools to gateway branch

---
 shared_resources/base_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 1402fbd..7fd08cd 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -80,7 +80,7 @@ dependencies:
     - climate-toolbox==0.1.5
     - noaa-coops==0.1.8
     - parameterize-jobs==0.1.1
-    - rhg_compute_tools==0.2.2
+    - git+https://github.com/RhodiumGroup/rhg_compute_tools.git@gateway#egg=rhg_compute_tools # needed for dask-gateway until merged into master
     - git+https://github.com/maritimeplanning/pytides.git@a8397a0e12645d22c62612385d18e63c4d67d6fa#egg=pytides
     - us==2.0.2 # installing from pip b/c conda thinks we need py 3.7
 name: base

From 74a3e5c0381d15e86fbd1ed35088b9dfb6abae24 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 17:35:11 -0700
Subject: [PATCH 203/224] 1st attempt at dask-gateway version of master

---
 .travis.yml                                |   6 +-
 notebook/Dockerfile                        |  61 ++------
 shared_resources/base_environment.yml      | 170 ++++++++++++---------
 shared_resources/scheduler_environment.yml |  10 +-
 worker/Dockerfile                          |  33 ----
 5 files changed, 113 insertions(+), 167 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 37de3ca..ef07493 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,8 +5,6 @@ env:
   jobs:
     - IMAGE_NAME=notebook
     - IMAGE_NAME=worker
-  global:
-    - CLAWPACK_COMMIT=5f6f94c6087c520e29fbdaf823965c2d8e2ec6b2
 
 install:
 - "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \
@@ -19,12 +17,10 @@ install:
 script:
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/scheduler:local -f Dockerfile_scheduler .; fi"
-- "travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT \
-    --build-arg CLAWPACK_COMMIT ."
+- travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT .
 - "if [[ $IMAGE_NAME == worker ]]; then docker build -t \
     rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \
     ../octave-worker; fi"
-- docker run -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT bash /clawpack/run_tests.sh
 
 deploy:
 - provider: script
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 01d632b..6481bc3 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,9 +1,6 @@
 FROM jupyter/base-notebook@sha256:6e276c402aa19799798b84b46e0e08fc03be193de9a81490158ca4d9026b8cae
 ARG DEBIAN_FRONTEND=noninteractive
 
-# pull in build args
-ARG CLAWPACK_COMMIT
-
 ## needed to make sure things with cython compile correctly
 ## (this will eventually become default in numpy)
 ENV NPY_DISTUTILS_APPEND_FLAGS=1
@@ -73,18 +70,17 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
     /opt/conda/bin/matio_73to5
 
 ## Set up jupyter lab extensions
-RUN jupyter labextension update --all 
-RUN jupyter labextension install \
-      @bokeh/jupyter_bokeh \
-      @jupyterlab/toc \
-      @jupyter-widgets/jupyterlab-manager \
-      @lckr/jupyterlab_variableinspector \
-      @pyviz/jupyterlab_pyviz \
-      @ryantam626/jupyterlab_code_formatter \
-      dask-labextension \
-      jupyterlab-plotly \
-      jupyterlab-tabular-data-editor \
-      plotlywidget
+RUN jupyter labextension update --all && \
+    jupyter labextension install \
+    @bokeh/jupyter_bokeh \
+    @jupyter-widgets/jupyterlab-manager \
+    @jupyter-widgets/jupyterlab-sidecar \
+    @pyviz/jupyterlab_pyviz \
+    dask-labextension \
+    jupyter-leaflet \
+    jupyter-matplotlib \
+    jupyterlab-plotly \
+    plotlywidget
 RUN jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
 
 ## configure nbdime
@@ -94,41 +90,6 @@ RUN source /opt/conda/etc/profile.d/conda.sh \
 # makes sure that the web-tools allow accessible IP
 COPY nbdime_config.json /opt/conda/etc/jupyter
 
-##################
-## clawpack added for coastal-specific image
-##################
-
-ENV CLAW=/clawpack
-ENV NETCDF4_DIR=/opt/conda
-ENV FC=gfortran
-ENV LIB_PATHS=/opt/conda/lib
-ENV CLAW_FFLAGS="-DNETCDF -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include \
-  -O3 -march=broadwell -ftree-vectorize -fPIC -fno-plt -ffunction-sections \
-  -fstack-protector-strong"
-
-# need to change shell in order for source command to work
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-# install clawpack (use root to access /clawpack and then fix permissions)
-USER root
-RUN source /opt/conda/etc/profile.d/conda.sh \
-  && conda activate \
-  && pip install --src=$(dirname $CLAW) -e \
-    git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
-RUN /tempdir/fix-permissions.sh /clawpack
-RUN /tempdir/fix-permissions.sh /opt/conda
-USER $NB_USER
-
-RUN source /opt/conda/etc/profile.d/conda.sh \
-  && conda activate \
-  && export EXE=$CLAW/geoclaw/xgeoclaw \
-  && export FFLAGS=$CLAW_FFLAGS \
-  && make -f /tempdir/Makefile.clawpack new \
-  && export EXE=$CLAW/geoclaw/xgeoclaw_omp \
-  && export FFLAGS="$FFLAGS -fopenmp" \
-  && make -f /tempdir/Makefile.clawpack new
-
-##################
 
 ## clean up
 RUN sudo rm -rf /var/lib/apt/lists/* /tempdir
diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index 7fd08cd..d23a18e 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -1,86 +1,108 @@
+name: base
 channels:
   - conda-forge
 dependencies:
-  - beautifulsoup4=4.9.1=py_1
-  - blas=2.17=openblas
-  - bottleneck=1.3.2=py38h8790de6_1
-  - boto3=1.14.56=pyh9f0ad1d_0
-  - bqplot=0.12.16=pyh9f0ad1d_0
-  - cachecontrol=0.12.6=py_0
-  - cartopy=0.18.0=py38h172510d_0
-  - cftime=1.2.1=py38h8790de6_0
-  - click=7.1.2=pyh9f0ad1d_0
-  - compilers=1.1.1=0
+  - beautifulsoup4=4.8.1=py37_0
+  - bottleneck=1.3.1=py37hc1659b7_0
+  - boto3=1.10.34=py_0
+  - bqplot=0.12.1=py_0
+  - bumpversion=0.5.3=py_1001
+  - cachecontrol=0.12.5=py_0
+  - cartopy=0.17.0=py37h423102d_1009
+  - cftime=1.0.4.2=py37hc1659b7_0
+  - click=7.0=py_0
+  - compilers=1.0.4=0
   - dask-glm=0.2.0=py_1
-  - dask-ml=1.6.0=py_0
-  - datashader=0.11.1=pyh9f0ad1d_0
+  - dask-ml=1.1.1=py_0
+  - datashader=0.8.0=py_0
   - descartes=1.1.0=py_4
-  - dropbox=10.1.1=pyh9f0ad1d_0
-  - esmpy=8.0.1=mpi_mpich_py38h08b7006_100
-  - fastparquet=0.4.1=py38h8790de6_0
-  - fiona=1.8.13=py38h033e0f6_1
+  - dropbox=9.4.0=py_0
+# need to make sure we get esmpy compiled with mpi otherwise xesmf regridding
+# won't work
+  - esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101 # pinkeep: esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101
+  - fastparquet=0.3.2=py37hc1659b7_0
+  - fiona=1.8.13=py37h900e953_0
   - fusepy=3.0.1=py_0
-  - fuzzywuzzy=0.17.0=py_0
-  - gcsfs=0.7.0=py_0
-  - gdal=3.0.4=py38h172510d_10
-  - geopandas=0.8.1=py_0
-  - geotiff=1.6.0=h05acad5_0
-  - geoviews=1.8.1=py_0
-  - git=2.28.0=pl526h5e3e691_1
-  - gitpython=3.1.8=py_0
-  - google-cloud-container=0.3.0=py38_0
-  - google-cloud-storage=1.31.0=pyh9f0ad1d_0
-  - holoviews=1.13.3=pyh9f0ad1d_0
-  - hvplot=0.6.0=pyh9f0ad1d_0
-  - h5netcdf=0.8.1=py_0
-  - intake-esm=2020.8.15=py_0
-  - jedi=0.17.2=py38h32f6830_0
-  - jellyfish=0.7.2=py38h1e0a361_1 # needed for "us" package
-  - jupyter-server-proxy=1.5.0=py_0 # need server proxy on workers if using remote scheduler
-  - kubernetes=1.18.8=0
-  - libmatio=1.5.17=hf54d351_0
-  - lxml=4.5.2=py38hbb43d70_0
-  - make=4.3=h516909a_0
-  - matplotlib=3.3.1=1
-  - nc-time-axis=1.2.0=py_1
-  - netcdf-fortran=4.5.3=mpi_mpich_h3923e1a_0
-  - netcdf4=1.5.4=nompi_py38hec8b9af_102
-  - nose=1.3.7=py38h32f6830_1004
-  - numba=0.51.2=py38hc5bc63f_0
-  - openpyxl=3.0.5=py_0
-  - phantomjs=2.1.1=1 # for geoviews
-  - pillow=7.2.0=py38h9776b28_1
-  - pip=20.2.2=py_0
-  - plotly=4.9.0=pyh9f0ad1d_0
-  - pygeos=0.8=py38hc7361b7_0
-  - pyinterp=0.3.2=mkl_py38hefd7b46_102
+# this gcc pin is necessary b/c of a weird feature in the h553295d_15 build
+# which makes it hard to build numpy-based cython extensions (like pyclaw).
+# we should try removing it whenever we next do an update and see if Clawpack
+# can still be built
+  - gcc_linux-64=7.3.0=h553295d_14 # pinkeep: gcc_linux-64=7.3.0=h553295d_14
+  - gcsfs=0.5.3=py_0
+  - gdal=3.0.2=py37hbb6b9fb_5
+  - geoalchemy2=0.6.3=py_0
+  - geopandas=0.6.2=py_0
+  - geopy=1.20.0=py_0
+  - geotiff=1.5.1=hbd99317_7
+  - geoviews=1.6.6=py_0
+  - git=2.24.0=pl526hce37bd2_1
+  - gitpython=3.0.5=py_0
+  - google-cloud-container=0.3.0=py37_0
+  - google-cloud-storage=1.23.0=py37_0
+  - holoviews=1.12.7=py_0
+  - h5netcdf=0.7.4=py_0
+  - icu=64.2=he1b5a44_1
+  - intake-esm=2020.6.11
+  - iris=2.2.0=py37_1003
+  - jedi=0.15.1=py37_0
+# need server proxy on workers if using remote scheduler
+  - jupyter-server-proxy=1.3.2=py_0
+  - kubernetes
+  - lapack=3.6.1=ha44fe06_2
+  - make=4.2.1=h14c3975_2004
+  - matplotlib=3.1.2=py37_1
+  - nc-time-axis=1.2.0=py_0
+  - ncurses=6.1=hf484d3e_1002
+  - netcdf-fortran=4.5.2=mpi_mpich_ha8580a0_2
+  - netcdf4=1.5.3=mpi_mpich_py37h01ee55b_1
+  - numba=0.46.0=py37hb3f55d8_1
+  - numcodecs=0.6.4=py37he1b5a44_0
+  # for geoviews
+  - phantomjs=2.1.1=1
+  - pip=19.3.1=py37_0
+  - plotly=4.3.0=py_0
+  - polyline=1.4.0=py_0
+  - pydap=3.2.2=py37_1000
+  - pygeos=0.5=py37h5d51c17_1
+  - pyinterp=0.0.7=py37h97f2665_0
   - pyshp=2.1.0=py_0
-  - python=3.8.5=h1103e12_7_cpython
-  - python-snappy=0.5.4=py38h7cfaab3_1
-  - pyyaml=5.3.1=py38h1e0a361_0
-  - rasterio=1.1.5=py38h033e0f6_1
-  - regionmask=0.6.1=py_0
-  - rtree=0.9.4=py38h08f867b_1
-  - scikit-image=0.17.2=py38hc5bc63f_2
-  - scikit-learn=0.23.2=py38hee58b96_0
-  - scipy=1.5.2=py38h8c5af15_0
-  - seaborn=0.10.1=1
-  - selenium=3.141.0=py38h1e0a361_1001 # for geoviews
-  - shapely=1.7.1=py38hc7361b7_0
-  - sparse=0.11.2=py_0
-  - statsmodels=0.12.0=py38h1e0a361_0
+  - python=3.7.3=h357f687_2 # pinkeep: python=3.7
+  - python-blosc=1.8.1=py37hf484d3e_0
+  - python-snappy=0.5.4=py37hee44bf9_1
+  - pyviz_comms=0.7.2=py_0
+  - pyyaml=5.2=py37h516909a_0
+  - rasterio=1.1.1=py37h900e953_0
+  - regionmask=0.4.0=py_0
+  - rtree=0.8.3=py37h7b0cdae_1003
+  - scikit-image=0.16.2=py37hb3f55d8_0
+  - scikit-learn=0.22=py37hcdab131_0
+  - scipy=1.3.2=py37h921218d_0
+  - seaborn=0.9.0=py_2
+  # for geoviews
+  - selenium=3.141.0=py37h516909a_1000
+  - shapely=1.6.4=py37h5d51c17_1007
+  - sparse=0.8.0=py_0
+  - statsmodels=0.10.2=py37hc1659b7_0
+  - unzip=6.0=h516909a_0
+  - uritemplate=3.0.0=py_1
   - xarray=0.16.0=py_0
-  - xarray-extras=0.4.2=py38h516909a_1000
-  - xesmf=0.3.0=py_0
+  - xesmf=0.2.1=py_0
+  - xgcm=0.2.0=py_0
   - xhistogram=0.1.1=py_0
-  - xlrd=1.2.0=pyh9f0ad1d_1
-  - zarr=2.4.0=py_0
-  - zeromq=4.3.2=he1b5a44_3
+  - xlrd=1.2.0=py_0
+  - xrft=0.2.0=py_0
+  - zarr=2.3.2=py37_0
+  - zeromq=4.3.2=he1b5a44_2
+  - zict=1.0.0=py_0
   - pip:
+    - mapbox==0.18.0
+    - py-noaa==1.0
+    - sidecar==0.3.0
     - climate-toolbox==0.1.5
-    - noaa-coops==0.1.8
+    - impactlab-tools==0.4.0
     - parameterize-jobs==0.1.1
-    - git+https://github.com/RhodiumGroup/rhg_compute_tools.git@gateway#egg=rhg_compute_tools # needed for dask-gateway until merged into master
-    - git+https://github.com/maritimeplanning/pytides.git@a8397a0e12645d22c62612385d18e63c4d67d6fa#egg=pytides
-    - us==2.0.2 # installing from pip b/c conda thinks we need py 3.7
-name: base
+    - rhg_compute_tools==0.2.2
+# need to install from master until 0.10.1
+# due to handling of remote scheduler
+# (we also should at some point switch to dask-gateway instead of dask-kubernetes)
+    - dask_kubernetes==0.10.1
\ No newline at end of file
diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
index cc60ed2..2bd36c9 100644
--- a/shared_resources/scheduler_environment.yml
+++ b/shared_resources/scheduler_environment.yml
@@ -4,12 +4,12 @@ channels:
     - conda-forge
 dependencies:
     - aiohttp=3.6.2=py38h516909a_0
-    - dask=2.25.0=py_0
-    - distributed=2.25.0=py38h32f6830_0
+    - dask=2.8.1=py_0
+    - distributed=2.8.1=py_0
     - dask-gateway=0.8.0=py38h32f6830_0
-    - lz4=3.1.0=py38h66f7c9e_0
-    - numpy=1.19.1=py38hbc27379_2
-    - pandas=1.1.1=py38h950e882_0
+    - lz4=2.2.1=py37hd79334b_0
+    - numpy=1.19.1=py37h8960a57_0
+    - pandas=0.25.3=py37hb3f55d8_0
     - tini=0.18.0=h14c3975_1001
 name: base
   
\ No newline at end of file
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 0390f52..a2e0497 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -5,39 +5,6 @@ ARG DEBIAN_FRONTEND=noninteractive
 RUN conda env update -f /opt/conda/specs/base_environment.yml
 RUN conda list -n base
 
-##################
-## clawpack added for coastal-specific image
-##################
-
-ARG CLAWPACK_COMMIT
-ENV CLAW=/clawpack
-ENV NETCDF4_DIR=/opt/conda
-ENV FC=gfortran
-ENV LIB_PATHS=/opt/conda/lib
-ENV CLAW_FFLAGS="-DNETCDF -L/opt/conda/lib -lnetcdf -lnetcdff -I/opt/conda/include \
-  -O3 -march=broadwell -ftree-vectorize -fPIC -fno-plt -ffunction-sections \
-  -fstack-protector-strong"
-
-# need to change shell in order for source command to work
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-# install clawpack
-RUN source /opt/conda/etc/profile.d/conda.sh \
-    && conda activate \
-    && pip install --src=$(dirname $CLAW) -e \
-        git+https://github.com/climateimpactlab/clawpack.git@${CLAWPACK_COMMIT}#egg=clawpack
-
-RUN source /opt/conda/etc/profile.d/conda.sh \
-    && conda activate \
-    && export EXE=$CLAW/geoclaw/xgeoclaw \
-    && export FFLAGS=$CLAW_FFLAGS \
-    && make -f /tempdir/Makefile.clawpack new \
-    && export EXE=$CLAW/geoclaw/xgeoclaw_omp \
-    && export FFLAGS="$FFLAGS -fopenmp" \
-    && make -f /tempdir/Makefile.clawpack new
-
-##################
-
 ## clean up
 RUN rm -rf /var/lib/apt/lists/* /tempdir \
     && conda clean -yaf \

From 4a9b012310d944b8fb99a74cbb08ad2382641daf Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 17:40:38 -0700
Subject: [PATCH 204/224] fix upstream images

---
 notebook/Dockerfile                        | 2 +-
 shared_resources/scheduler_environment.yml | 7 +++----
 worker/Dockerfile_scheduler                | 2 +-
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 6481bc3..0034b26 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook@sha256:6e276c402aa19799798b84b46e0e08fc03be193de9a81490158ca4d9026b8cae
+FROM jupyter/base-notebook:notebook-6.0.0
 ARG DEBIAN_FRONTEND=noninteractive
 
 ## needed to make sure things with cython compile correctly
diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
index 2bd36c9..31e1841 100644
--- a/shared_resources/scheduler_environment.yml
+++ b/shared_resources/scheduler_environment.yml
@@ -3,13 +3,12 @@
 channels:
     - conda-forge
 dependencies:
-    - aiohttp=3.6.2=py38h516909a_0
+    - aiohttp=3.6.2=py37h1de35cc_0
     - dask=2.8.1=py_0
     - distributed=2.8.1=py_0
-    - dask-gateway=0.8.0=py38h32f6830_0
+    - dask-gateway=0.8.0=py37hc8dfbb8_0
     - lz4=2.2.1=py37hd79334b_0
     - numpy=1.19.1=py37h8960a57_0
     - pandas=0.25.3=py37hb3f55d8_0
     - tini=0.18.0=h14c3975_1001
-name: base
-  
\ No newline at end of file
+name: base
\ No newline at end of file
diff --git a/worker/Dockerfile_scheduler b/worker/Dockerfile_scheduler
index 0c29a06..b3fc704 100644
--- a/worker/Dockerfile_scheduler
+++ b/worker/Dockerfile_scheduler
@@ -1,5 +1,5 @@
 ## using same base image as the "base image for the base image" of the notebook image
-FROM ubuntu:focal-20200703@sha256:d5a6519d9f048100123c568eb83f7ef5bfcad69b01424f420f17c932b00dea76
+FROM ubuntu:bionic-20190612@sha256:6e9f67fa63b0323e9a1e587fd71c561ba48a034504fb804fd26fd8800039835d
 ARG DEBIAN_FRONTEND=noninteractive
 
 # needed to properly install packages that use numpy libraries

From 88ca8ad90fa90b5bdd7a89dd3f9bf629a7a6c58c Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 17:46:34 -0700
Subject: [PATCH 205/224] update rhg_compute_tools w/ dask-gateway

---
 shared_resources/base_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index d23a18e..fbe6616 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -101,7 +101,7 @@ dependencies:
     - climate-toolbox==0.1.5
     - impactlab-tools==0.4.0
     - parameterize-jobs==0.1.1
-    - rhg_compute_tools==0.2.2
+    - git+https://github.com/RhodiumGroup/rhg_compute_tools.git@gateway#egg=rhg_compute_tools # needed for dask-gateway until merged into master
 # need to install from master until 0.10.1
 # due to handling of remote scheduler
 # (we also should at some point switch to dask-gateway instead of dask-kubernetes)

From cfa0dbdfe87096045bd0f5f42d91d8fe0fe5c460 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 17:56:23 -0700
Subject: [PATCH 206/224] add r env back

---
 notebook/Dockerfile                        | 4 ++++
 pin.py                                     | 5 ++++-
 shared_resources/scheduler_environment.yml | 2 +-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 0034b26..f8b10eb 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -43,6 +43,7 @@ RUN mkdir /opt/conda/specs
 COPY shared_resources/base_environment.yml /opt/conda/specs
 COPY shared_resources/scheduler_environment.yml /opt/conda/specs
 COPY notebook_environment.yml /opt/conda/specs
+COPY r_environment.yml /opt/conda/specs
 RUN conda config --add channels conda-forge && \
   conda config --set channel_priority strict
 
@@ -61,6 +62,9 @@ USER $NB_USER
 # update environment with nb-specific packages
 RUN conda env update -f /opt/conda/specs/notebook_environment.yml
 
+# add r env
+RUN conda env create -f /opt/conda/specs/r_environment.yml
+
 RUN conda list -n base
 
 # Compile MatI/O
diff --git a/pin.py b/pin.py
index 68dc324..37ce9eb 100644
--- a/pin.py
+++ b/pin.py
@@ -30,6 +30,7 @@
     ('shared_resources/scheduler_environment.yml', 'base'),
     ('notebook/notebook_environment.yml', 'base'),
     ('octave-worker/octave_environment.yml', 'base'),
+    ('notebook/r_environment.yml', 'r'),
 ]
 
 
@@ -339,10 +340,12 @@ def unpin(file, dry_run):
         unpin_files([SPEC_FILES[2]], dry_run=dry_run)
     elif file == 'octave':
         unpin_files([SPEC_FILES[3]], dry_run=dry_run)
+    elif file == 'r':
+        unpin_files([SPEC_FILES[4]], dry_run=dry_run)
     else:
         raise ValueError(
             'env type not recognized: {}'
-            'choose from "base", "scheduler", "notebook", "octave", or "all".'
+            'choose from "base", "scheduler", "notebook", "octave", "r", or "all".'
             .format(file))
 
 
diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
index 31e1841..66149b3 100644
--- a/shared_resources/scheduler_environment.yml
+++ b/shared_resources/scheduler_environment.yml
@@ -3,7 +3,7 @@
 channels:
     - conda-forge
 dependencies:
-    - aiohttp=3.6.2=py37h1de35cc_0
+    - aiohttp=3.6.2=py37h7b6447c_0
     - dask=2.8.1=py_0
     - distributed=2.8.1=py_0
     - dask-gateway=0.8.0=py37hc8dfbb8_0

From 3b894165efe2b9e99fe08362f2d46bf0167157cd Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 17:58:35 -0700
Subject: [PATCH 207/224] drop matio

---
 notebook/Dockerfile            |  6 ------
 octave-worker/Dockerfile       |  6 ------
 shared_resources/matio_73to5.c | 31 -------------------------------
 3 files changed, 43 deletions(-)
 delete mode 100644 shared_resources/matio_73to5.c

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index f8b10eb..84d35d8 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -67,12 +67,6 @@ RUN conda env create -f /opt/conda/specs/r_environment.yml
 
 RUN conda list -n base
 
-# Compile MatI/O
-RUN source /opt/conda/etc/profile.d/conda.sh \
-  && conda activate \
-  && g++ $CFLAGS -L/opt/conda/lib -lmatio /tempdir/matio_73to5.c -o \
-    /opt/conda/bin/matio_73to5
-
 ## Set up jupyter lab extensions
 RUN jupyter labextension update --all && \
     jupyter labextension install \
diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile
index 0b72b44..af386d1 100644
--- a/octave-worker/Dockerfile
+++ b/octave-worker/Dockerfile
@@ -14,12 +14,6 @@ RUN apt-get update \
 # add octave-specific packages
 RUN conda env update -f /opt/conda/specs/octave_environment.yml
 
-# Compile MatI/O
-RUN source /opt/conda/etc/profile.d/conda.sh \
-  && conda activate \
-  && g++ $CFLAGS -L/opt/conda/lib -lmatio /tempdir/matio_73to5.c -o \
-    /opt/conda/bin/matio_73to5
-
 RUN rm -rf /tempdir \
     && conda clean -yaf \
     && sudo apt-get clean
diff --git a/shared_resources/matio_73to5.c b/shared_resources/matio_73to5.c
deleted file mode 100644
index 058a3b7..0000000
--- a/shared_resources/matio_73to5.c
+++ /dev/null
@@ -1,31 +0,0 @@
-#include <stdlib.h>
-#include <stdio.h>
-#include "matio.h"
-
-int main(int argc, char **argv) {
-    mat_t    *matfp_in;
-    mat_t    *matfp_out;
-    matvar_t *matvar;
-    matfp_in = Mat_Open(argv[1], MAT_ACC_RDONLY);
-    if ( NULL == matfp_in ) {
-        fprintf(stderr, "Error opening MAT file \"%s\"!\n", argv[1]);
-        return EXIT_FAILURE;
-    }
-    matfp_out = Mat_CreateVer(argv[2], NULL, MAT_FT_MAT5);
-
-    while ( (matvar = Mat_VarReadNext(matfp_in)) != NULL ) {
-        printf("Writing MAT variable: %s... ", matvar->name);
-        if (0 != Mat_VarWrite(matfp_out, matvar, MAT_COMPRESSION_NONE)) {
-            printf("failed.\n");
-        } else {
-            printf("succeeded.\n");
-        }
-        Mat_VarFree(matvar);
-    }
-    
-    Mat_Close(matfp_out);
-    fprintf(stderr, "Wrote MAT file: \"%s\"\n", argv[2]);
-    
-    Mat_Close(matfp_in);
-    return EXIT_SUCCESS;
-}

From 4f36bcc216a0b85af0ede0356d49668a8c7737a6 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:00:16 -0700
Subject: [PATCH 208/224] revert octave version

---
 notebook/notebook_environment.yml    | 4 ++--
 octave-worker/octave_environment.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index d7a8694..180871c 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -22,8 +22,8 @@ dependencies:
   - nano
   - nbdime
   - nb_conda_kernels
-  - octave_kernel=0.32.0=pyh9f0ad1d_0
-  - oct2py=5.2.0=pyh9f0ad1d_0
+  - octave_kernel=0.31.0=py_0
+  - oct2py=5.0.4=py_0
   - openssh
   - papermill
   - pip
diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml
index 78742ad..704df48 100755
--- a/octave-worker/octave_environment.yml
+++ b/octave-worker/octave_environment.yml
@@ -2,5 +2,5 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - octave_kernel=0.32.0=pyh9f0ad1d_0
-  - oct2py=5.2.0=pyh9f0ad1d_0
+  - octave_kernel=0.31.0=py_0
+  - oct2py=5.0.4=py_0

From 70d51d49e1246fbb63c6db41fdf50006c06e49aa Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:04:40 -0700
Subject: [PATCH 209/224] add tabular data editor

---
 notebook/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 84d35d8..024ba7d 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -78,6 +78,7 @@ RUN jupyter labextension update --all && \
     jupyter-leaflet \
     jupyter-matplotlib \
     jupyterlab-plotly \
+    jupyterlab-tabular-data-editor \
     plotlywidget
 RUN jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
 

From 5e8c8b8bdbd014ee076bd3ace640e9ee1fe88c4c Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:06:34 -0700
Subject: [PATCH 210/224] drop clawpack makefile

---
 shared_resources/Makefile.clawpack | 58 ------------------------------
 1 file changed, 58 deletions(-)
 delete mode 100755 shared_resources/Makefile.clawpack

diff --git a/shared_resources/Makefile.clawpack b/shared_resources/Makefile.clawpack
deleted file mode 100755
index b0078fa..0000000
--- a/shared_resources/Makefile.clawpack
+++ /dev/null
@@ -1,58 +0,0 @@
-# Makefile for Clawpack code in this directory.
-# This version only sets the local files and frequently changed
-# options, and then includes the standard makefile pointed to by CLAWMAKE.
-CLAWMAKE ?= $(CLAW)/clawutil/src/Makefile.common
-
-# See the above file for details and a list of make options, or type
-#   make .help
-# at the unix prompt.
-
-
-# Adjust these variables if desired:
-# ----------------------------------
-
-CLAW_PKG = geoclaw                  # Clawpack package to use
-EXE ?= $(CLAW)/geoclaw/xgeoclaw     # Executable to create/use
-SETRUN_FILE = setrun.py        # File containing function to make data
-OUTDIR = _output               # Directory for output
-SETPLOT_FILE = setplot.py      # File containing function to set plots
-PLOTDIR = _plots               # Directory for plots
-
-# Environment variable FC should be set to fortran compiler, e.g. gfortran
-FFLAGS ?=
-
-# ---------------------------------
-# package sources for this program:
-# ---------------------------------
-
-GEOLIB = $(CLAW)/geoclaw/src/2d/shallow
-include $(GEOLIB)/Makefile.geoclaw
-
-# ---------------------------------------
-# package sources specifically to exclude
-# (i.e. if a custom replacement source
-#  under a different name is provided)
-# ---------------------------------------
-
-EXCLUDE_MODULES = \
-
-EXCLUDE_SOURCES = \
-
-# ----------------------------------------
-# List of custom sources for this program:
-# ----------------------------------------
-
-RIEMANN = $(CLAW)/riemann/src
-
-MODULES = \
-
-SOURCES = \
-  $(RIEMANN)/rpn2_geoclaw.f \
-  $(RIEMANN)/rpt2_geoclaw.f \
-  $(RIEMANN)/geoclaw_riemann_utils.f \
-
-#-------------------------------------------------------------------
-# Include Makefile containing standard definitions and make options:
-include $(CLAWMAKE)
-
-### DO NOT remove this line - make depends on it ###

From 86b4d636cda1132439fb2c98e46b60c767d8484f Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:07:30 -0700
Subject: [PATCH 211/224] drop dask-kubernetes

---
 shared_resources/base_environment.yml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index fbe6616..d62c912 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -101,8 +101,4 @@ dependencies:
     - climate-toolbox==0.1.5
     - impactlab-tools==0.4.0
     - parameterize-jobs==0.1.1
-    - git+https://github.com/RhodiumGroup/rhg_compute_tools.git@gateway#egg=rhg_compute_tools # needed for dask-gateway until merged into master
-# need to install from master until 0.10.1
-# due to handling of remote scheduler
-# (we also should at some point switch to dask-gateway instead of dask-kubernetes)
-    - dask_kubernetes==0.10.1
\ No newline at end of file
+    - git+https://github.com/RhodiumGroup/rhg_compute_tools.git@gateway#egg=rhg_compute_tools # needed for dask-gateway until merged into master
\ No newline at end of file

From d09de678ef38776b7012fcd09d604d8badbe6f1f Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:09:15 -0700
Subject: [PATCH 212/224] drop vdatum

---
 shared_resources/common.sh | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/shared_resources/common.sh b/shared_resources/common.sh
index c130c3a..21e2eb8 100755
--- a/shared_resources/common.sh
+++ b/shared_resources/common.sh
@@ -8,7 +8,6 @@ apt-get install -yq --no-install-recommends \
   unzip \
   ca-certificates \
   curl \
-  default-jre \
   locales \
   lsb-release \
   gnupg2 \
@@ -37,14 +36,6 @@ apt-get clean
 wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy
 chmod +x /usr/bin/cloud_sql_proxy
 
-# download vdatum
-wget https://vdatum.noaa.gov/download/data/vdatum_v4.1.zip -O /usr/bin/vdatum.zip
-wget https://vdatum.noaa.gov/download/data/vdatum_EGM1996.zip -O /usr/bin/vdatum_EGM1996.zip
-unzip /usr/bin/vdatum.zip -d /usr/bin
-unzip -o /usr/bin/vdatum_EGM1996.zip -d /usr/bin
-rm /usr/bin/vdatum.zip
-rm /usr/bin/vdatum_EGM1996.zip
-
 # filepath curating
 chmod +x /usr/bin/prepare.sh
 mkdir /gcs

From f751bbb4f2c00013f0dba618730bd67dc97a251b Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:10:47 -0700
Subject: [PATCH 213/224] drop ulimit change needed for clawpack

---
 worker/prepare.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/worker/prepare.sh b/worker/prepare.sh
index 70acb9e..90ebc0e 100644
--- a/worker/prepare.sh
+++ b/worker/prepare.sh
@@ -42,8 +42,5 @@ if [[ "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then
     gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS;
 fi
 
-# needed for CLAWPACK to not throw segfaults sometimes
-ulimit -s unlimited
-
 # Run extra commands
 $@

From 969b0f12c0386068c3f21eb17cfdc16e4156d8dc Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:15:25 -0700
Subject: [PATCH 214/224] fix conda to py 3.7

---
 worker/Dockerfile_scheduler | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/worker/Dockerfile_scheduler b/worker/Dockerfile_scheduler
index b3fc704..bcff8d1 100644
--- a/worker/Dockerfile_scheduler
+++ b/worker/Dockerfile_scheduler
@@ -29,15 +29,14 @@ ENV CONDA_DIR=/opt/conda \
     LANGUAGE=en_US.UTF-8
 ENV PATH=$CONDA_DIR/bin:$PATH
 
-ENV MINICONDA_VERSION=4.8.3 \
-    MINICONDA_MD5=d63adf39f2c220950a063e0529d4ff74 \
-    CONDA_VERSION=4.8.3
+ENV MINICONDA_VERSION=4.7.12.1 \
+    MINICONDA_MD5=81c773ff87af5cfac79ab862942ab6b3
 
 WORKDIR /tmp
-RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
-    echo "${MINICONDA_MD5} *Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
-    /bin/bash Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
-    rm Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    echo "${MINICONDA_MD5} *Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
+    /bin/bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \
     conda config --system --prepend channels conda-forge && \
     conda config --system --set channel_priority strict && \
     conda update -n base --yes conda && \

From 6d1e121cd85eaefeb7813bef53291857378677fc Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:34:01 -0700
Subject: [PATCH 215/224] make scheduler lighter weight w/o  common installs

---
 worker/Dockerfile           |  3 +++
 worker/Dockerfile_scheduler | 11 ++++++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/worker/Dockerfile b/worker/Dockerfile
index a2e0497..8981dba 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -2,6 +2,9 @@
 FROM rhodium/scheduler:local
 ARG DEBIAN_FRONTEND=noninteractive
 
+## perform a bunch of common actions
+RUN bash /tempdir/common.sh
+
 RUN conda env update -f /opt/conda/specs/base_environment.yml
 RUN conda list -n base
 
diff --git a/worker/Dockerfile_scheduler b/worker/Dockerfile_scheduler
index bcff8d1..0d937f3 100644
--- a/worker/Dockerfile_scheduler
+++ b/worker/Dockerfile_scheduler
@@ -13,8 +13,13 @@ COPY shared_resources /tempdir
 COPY add_service_creds.py /usr/bin
 COPY prepare.sh /usr/bin
 
-## perform a bunch of common actions
-RUN bash /tempdir/common.sh
+# install apt-get packages
+apt-get update -y --no-install-recommends
+apt-get install -yq --no-install-recommends \
+  locales \
+  sudo \
+  wget
+
 
 ###########
 ## install miniconda (following jupyter/base-notebook
@@ -50,7 +55,7 @@ COPY shared_resources/base_environment.yml /opt/conda/specs
 RUN conda env update -f /opt/conda/specs/scheduler_environment.yml
 
 ## clean up
-RUN rm -rf /var/lib/apt/lists/* /tmp/*\
+RUN rm -rf /var/lib/apt/lists/* /tmp/* \
     && conda clean -yaf \
     && sudo apt-get clean
 

From 9204936fcee08d9df11e6b74e554b2c1b43b5309 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:40:11 -0700
Subject: [PATCH 216/224] fix apt-get in new scheduler dockerfile

---
 worker/Dockerfile_scheduler | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/worker/Dockerfile_scheduler b/worker/Dockerfile_scheduler
index 0d937f3..fcbd1a7 100644
--- a/worker/Dockerfile_scheduler
+++ b/worker/Dockerfile_scheduler
@@ -14,8 +14,8 @@ COPY add_service_creds.py /usr/bin
 COPY prepare.sh /usr/bin
 
 # install apt-get packages
-apt-get update -y --no-install-recommends
-apt-get install -yq --no-install-recommends \
+RUN apt-get update -y --no-install-recommends
+RUN apt-get install -yq --no-install-recommends \
   locales \
   sudo \
   wget

From 28b2bb742861c55ef32fc233689285b2059c59d3 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:46:52 -0700
Subject: [PATCH 217/224] add ca-certificates back

---
 worker/Dockerfile_scheduler | 1 +
 1 file changed, 1 insertion(+)

diff --git a/worker/Dockerfile_scheduler b/worker/Dockerfile_scheduler
index fcbd1a7..6db3eda 100644
--- a/worker/Dockerfile_scheduler
+++ b/worker/Dockerfile_scheduler
@@ -16,6 +16,7 @@ COPY prepare.sh /usr/bin
 # install apt-get packages
 RUN apt-get update -y --no-install-recommends
 RUN apt-get install -yq --no-install-recommends \
+  ca-certificates \
   locales \
   sudo \
   wget

From c2ffce3815f2c2ca73f243fc2c9b75be232fcb5d Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Sep 2020 18:56:52 -0700
Subject: [PATCH 218/224] fix scheduler

---
 shared_resources/scheduler_environment.yml | 4 ++--
 worker/Dockerfile_scheduler                | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
index 66149b3..91656c2 100644
--- a/shared_resources/scheduler_environment.yml
+++ b/shared_resources/scheduler_environment.yml
@@ -3,12 +3,12 @@
 channels:
     - conda-forge
 dependencies:
-    - aiohttp=3.6.2=py37h7b6447c_0
+    - aiohttp=3.6.2=py37h516909a_0
     - dask=2.8.1=py_0
     - distributed=2.8.1=py_0
     - dask-gateway=0.8.0=py37hc8dfbb8_0
     - lz4=2.2.1=py37hd79334b_0
-    - numpy=1.19.1=py37h8960a57_0
+    - numpy=1.19.1=py37h7ea13bd_2
     - pandas=0.25.3=py37hb3f55d8_0
     - tini=0.18.0=h14c3975_1001
 name: base
\ No newline at end of file
diff --git a/worker/Dockerfile_scheduler b/worker/Dockerfile_scheduler
index 6db3eda..2a0030c 100644
--- a/worker/Dockerfile_scheduler
+++ b/worker/Dockerfile_scheduler
@@ -67,4 +67,6 @@ ENV OPENBLAS_NUM_THREADS=1
 
 WORKDIR /
 
+RUN chmod +x /usr/bin/prepare.sh
+
 ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
\ No newline at end of file

From 4c82d473dae07984ae5fffabf50319a73198867d Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Thu, 10 Sep 2020 09:02:04 -0700
Subject: [PATCH 219/224] drop gcc pin and update compilers package

---
 shared_resources/base_environment.yml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index d62c912..e37dd70 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -11,7 +11,7 @@ dependencies:
   - cartopy=0.17.0=py37h423102d_1009
   - cftime=1.0.4.2=py37hc1659b7_0
   - click=7.0=py_0
-  - compilers=1.0.4=0
+  - compilers=1.1.1=0
   - dask-glm=0.2.0=py_1
   - dask-ml=1.1.1=py_0
   - datashader=0.8.0=py_0
@@ -23,11 +23,6 @@ dependencies:
   - fastparquet=0.3.2=py37hc1659b7_0
   - fiona=1.8.13=py37h900e953_0
   - fusepy=3.0.1=py_0
-# this gcc pin is necessary b/c of a weird feature in the h553295d_15 build
-# which makes it hard to build numpy-based cython extensions (like pyclaw).
-# we should try removing it whenever we next do an update and see if Clawpack
-# can still be built
-  - gcc_linux-64=7.3.0=h553295d_14 # pinkeep: gcc_linux-64=7.3.0=h553295d_14
   - gcsfs=0.5.3=py_0
   - gdal=3.0.2=py37hbb6b9fb_5
   - geoalchemy2=0.6.3=py_0
@@ -45,8 +40,6 @@ dependencies:
   - intake-esm=2020.6.11
   - iris=2.2.0=py37_1003
   - jedi=0.15.1=py37_0
-# need server proxy on workers if using remote scheduler
-  - jupyter-server-proxy=1.3.2=py_0
   - kubernetes
   - lapack=3.6.1=ha44fe06_2
   - make=4.2.1=h14c3975_2004

From 929a2b24d979369b8d28631eddb4554fc77a53ee Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Thu, 10 Sep 2020 09:17:55 -0700
Subject: [PATCH 220/224] add missing r_env file

---
 notebook/r_environment.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 notebook/r_environment.yml

diff --git a/notebook/r_environment.yml b/notebook/r_environment.yml
new file mode 100644
index 0000000..09e3e50
--- /dev/null
+++ b/notebook/r_environment.yml
@@ -0,0 +1,24 @@
+name: r
+channels:
+  - conda-forge
+  - r
+dependencies:
+  - rpy2=3.1.0=py38r36hc1659b7_3
+  - r-base=3.6.1=h3a67422_6
+  - r-irkernel=1.1=r36h6115d3f_0
+  - r-plyr=1.8.4=r36h0357c0b_1003
+  - r-devtools=2.2.1=r36h6115d3f_0
+  - r-tidyverse=1.3.0=r36h6115d3f_0
+  - r-shiny=1.4.0=r36h6115d3f_0
+  - r-rmarkdown=1.18=r36h6115d3f_0
+  - r-forecast=8.10=r36h0357c0b_0
+  - r-rsqlite=2.1.4=r36h0357c0b_0
+  - r-reshape2=1.4.3=r36h0357c0b_1004
+  - r-nycflights13=1.0.1=r36h6115d3f_0
+  - r-caret=6.0_84=r36hcdcec82_1
+  - r-rcurl=1.95_4.12=r36hcdcec82_1
+  - r-crayon=1.3.4=r36h6115d3f_1002
+  - r-randomforest=4.6_14=r36h9bbef5b_1002
+  - r-sparklyr=1.0.5=r36h6115d3f_0
+  - r-htmlwidgets=1.5.1=r36h6115d3f_0
+  - r-hexbin=1.28.0=r36h9bbef5b_0

From 687f977b169700e7dfde1b002c0f86694e613ee4 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Thu, 10 Sep 2020 09:53:57 -0700
Subject: [PATCH 221/224] set shell to bash in notebook dockerfile

---
 notebook/Dockerfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/notebook/Dockerfile b/notebook/Dockerfile
index 024ba7d..ad4c39d 100755
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,6 +1,9 @@
 FROM jupyter/base-notebook:notebook-6.0.0
 ARG DEBIAN_FRONTEND=noninteractive
 
+# set shell to bash so that later can use source
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 ## needed to make sure things with cython compile correctly
 ## (this will eventually become default in numpy)
 ENV NPY_DISTUTILS_APPEND_FLAGS=1

From 3081702050a67d88a7ab33aaad63bf221dbbe54e Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Thu, 10 Sep 2020 11:27:31 -0700
Subject: [PATCH 222/224] bump gitpython b/c nbdime needs gitdb

---
 shared_resources/base_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared_resources/base_environment.yml b/shared_resources/base_environment.yml
index e37dd70..990c316 100755
--- a/shared_resources/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -31,7 +31,7 @@ dependencies:
   - geotiff=1.5.1=hbd99317_7
   - geoviews=1.6.6=py_0
   - git=2.24.0=pl526hce37bd2_1
-  - gitpython=3.0.5=py_0
+  - gitpython=3.1.8=py_0
   - google-cloud-container=0.3.0=py37_0
   - google-cloud-storage=1.23.0=py37_0
   - holoviews=1.12.7=py_0

From 1b99b284bd990c5bf808e23a237ba9975d022fd8 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Mon, 9 Nov 2020 18:10:48 -0800
Subject: [PATCH 223/224] bump jupyterhub

---
 notebook/notebook_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
index 180871c..9d9d810 100755
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -13,7 +13,7 @@ dependencies:
   - ipypublish
   - isort
 # need jupyterhub version to match that running on the hub
-  - jupyterhub=1.1.0 # pinkeep: jupyterhub=1.1.0
+  - jupyterhub=1.2.1 # pinkeep: jupyterhub=1.2.1
   - jupyterlab
   - jupyterlab-git
   - jupyterlab_code_formatter

From abc3f973242e7852c44c4958f267060ef7b7ad29 Mon Sep 17 00:00:00 2001
From: Ian Bolliger <ibolliger@rhg.com>
Date: Wed, 9 Dec 2020 14:28:37 -0800
Subject: [PATCH 224/224] bump pgks needed for scheduler

---
 .gitignore                                 |  3 ++-
 shared_resources/scheduler_environment.yml | 19 ++++++++++---------
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/.gitignore b/.gitignore
index 148d7d7..15103b1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,5 @@ worker-config.yml
 __pycache__
 .pytest_cache
 .vscode/settings.json
-.devcontainer.json
\ No newline at end of file
+.devcontainer.json
+.DS_Store
diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
index 91656c2..f08e3bc 100644
--- a/shared_resources/scheduler_environment.yml
+++ b/shared_resources/scheduler_environment.yml
@@ -1,14 +1,15 @@
 # Only the packages necessary to build a dask-gateway scheduler. As seen here:
 # https://github.com/dask/dask-gateway/blob/master/dask-gateway/Dockerfile
 channels:
-    - conda-forge
+  - conda-forge
 dependencies:
-    - aiohttp=3.6.2=py37h516909a_0
-    - dask=2.8.1=py_0
-    - distributed=2.8.1=py_0
-    - dask-gateway=0.8.0=py37hc8dfbb8_0
-    - lz4=2.2.1=py37hd79334b_0
-    - numpy=1.19.1=py37h7ea13bd_2
-    - pandas=0.25.3=py37hb3f55d8_0
-    - tini=0.18.0=h14c3975_1001
+  - aiohttp=3.7.3=py38h25fe258_0
+  - dask=2.30.0=py_0
+  - distributed=2.30.1=py38h578d9bd_0
+  - dask-gateway=0.9.0=py38h578d9bd_0
+  - lz4=3.1.1=py38h87b837d_0
+  - jupyter-server-proxy=1.5.0=py_0
+  - numpy=1.19.4=py38hf0fd68c_1
+  - pandas=1.1.4=py38h0ef3d22_0
+  - tini=0.18.0=h14c3975_1001
 name: base
\ No newline at end of file