diff --git a/.gitignore b/.gitignore index 79e93af..15103b1 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ worker-config.yml .coverage __pycache__ .pytest_cache +.vscode/settings.json +.devcontainer.json +.DS_Store diff --git a/.travis.yml b/.travis.yml index 7032e46..ef07493 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,26 @@ -sudo: required - services: - docker env: -- IMAGE_NAME=notebook -- IMAGE_NAME=worker + jobs: + - IMAGE_NAME=notebook + - IMAGE_NAME=worker install: -- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak \ -'s/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' \ -notebook/worker-template.yml; else sed -i.bak \ -'s/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' \ -notebook/worker-template.yml; fi" -- "rm notebook/worker-template.yml.bak" -- "cat notebook/worker-template.yml | grep image:" -- "cp base_environment.yml $IMAGE_NAME/base_environment.yml" -- "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh" -- "cd $IMAGE_NAME" +- "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \ + $IMAGE_NAME/shared_resources" +- "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \ + octave-worker/shared_resources && chmod -R +x octave-worker/shared_resources; fi" +- cd $IMAGE_NAME script: -- docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT . +- "if [[ $IMAGE_NAME == worker ]]; then docker build -t \ + rhodium/scheduler:local -f Dockerfile_scheduler .; fi" +- travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT . +- "if [[ $IMAGE_NAME == worker ]]; then docker build -t \ + rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \ + ../octave-worker; fi" deploy: - provider: script @@ -31,10 +30,9 @@ deploy: docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" - skip_cleanup: true on: all_branches: true - condition: $TRAVIS_BRANCH =~ ^dev + condition: $TRAVIS_BRANCH != master - provider: script script: >- @@ -44,7 +42,6 @@ deploy: docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" - skip_cleanup: true on: branch: master @@ -54,25 +51,51 @@ deploy: rhodium/$IMAGE_NAME:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" - skip_cleanup: true on: tags: true +# scheduler builds +- provider: script + script: >- + docker tag rhodium/scheduler:local rhodium/scheduler:$TRAVIS_COMMIT && + docker tag rhodium/scheduler:local rhodium/scheduler:$TRAVIS_BRANCH && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/scheduler:$TRAVIS_COMMIT" && + docker push "rhodium/scheduler:$TRAVIS_BRANCH" + on: + all_branches: true + condition: ($TRAVIS_BRANCH != master) && ($IMAGE_NAME == worker) + +- provider: script + script: >- + docker tag rhodium/scheduler:local rhodium/scheduler:latest && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/scheduler:$TRAVIS_COMMIT" && + docker push "rhodium/scheduler:latest" + on: + branch: master + condition: $IMAGE_NAME = worker + +- provider: script + script: >- + docker tag rhodium/scheduler:local rhodium/scheduler:$TRAVIS_TAG && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/scheduler:$TRAVIS_TAG" + on: + tags: true + condition: $IMAGE_NAME = worker + # octave-worker builds - provider: script script: >- - docker build -t rhodium/octave-worker:$TRAVIS_COMMIT - --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT ../octave-worker && docker tag rhodium/octave-worker:$TRAVIS_COMMIT rhodium/octave-worker:$TRAVIS_BRANCH && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && docker push "rhodium/octave-worker:$TRAVIS_BRANCH" - skip_cleanup: true on: all_branches: true - condition: $TRAVIS_BRANCH =~ ^dev - condition: $IMAGE_NAME = worker + condition: ($TRAVIS_BRANCH != master) && ($IMAGE_NAME == worker) - provider: script script: >- @@ -84,7 +107,6 @@ deploy: docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && docker push "rhodium/octave-worker:dev" && docker push "rhodium/octave-worker:latest" - skip_cleanup: true on: branch: master condition: $IMAGE_NAME = worker @@ -95,22 +117,6 @@ deploy: rhodium/octave-worker:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/octave-worker:$TRAVIS_TAG" - skip_cleanup: true on: tags: true - condition: $IMAGE_NAME = worker - - # - stage: alignment - # language: python - # python: - # - 3.6 - # script: - # - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - # - bash miniconda.sh -b -p $HOME/miniconda - # - export PATH="$HOME/miniconda/bin:$PATH" - # - hash -r - # - conda config --set always_yes yes --set changeps1 no - # - conda update -q conda - # - conda info -a - # - conda install pytest pytest-cov pyyaml - # - pytest + condition: $IMAGE_NAME = worker \ No newline at end of file diff --git a/bump.py b/bump.py index 42297a5..5a14384 100644 --- a/bump.py +++ b/bump.py @@ -5,7 +5,6 @@ SEARCH_PATTERNS = [ ('.travis.yml', r'(?P
.*TAG=)(?P\d{4}-\d{2}-\d{2})\.?(?P\d{2})?(?P[.\s]*)$'),
-    ('notebook/worker-template.yml', r'(?P
.*image: rhodium/worker:)(?P\d{4}-\d{2}-\d{2})\.?(?P\d{2})?(?P[.\s]*)$'),
     ('jupyter-config.yml', r'(?P
.*tag: )(?P\d{4}-\d{2}-\d{2})\.?(?P\d{2})?(?P[.\s]*)$')]
 
 
diff --git a/notebook/Dockerfile b/notebook/Dockerfile
old mode 100644
new mode 100755
index aad9f6d..ad4c39d
--- a/notebook/Dockerfile
+++ b/notebook/Dockerfile
@@ -1,6 +1,9 @@
 FROM jupyter/base-notebook:notebook-6.0.0
 ARG DEBIAN_FRONTEND=noninteractive
 
+# set shell to bash so that later can use source
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 ## needed to make sure things with cython compile correctly
 ## (this will eventually become default in numpy)
 ENV NPY_DISTUTILS_APPEND_FLAGS=1
@@ -18,13 +21,9 @@ USER $NB_USER
 
 ## filepath curation
 RUN sudo mkdir /pre-home && sudo chown -R $NB_USER /pre-home
-COPY worker-template.yml /pre-home
-COPY add_service_creds.py /pre-home
-COPY run_sql_proxy.py /pre-home
-COPY config.yaml /pre-home
+COPY config.yaml set_gateway_opts.py /pre-home/
 
-RUN sudo mkdir /tempdir
-COPY common.sh /tempdir
+COPY shared_resources /tempdir
 
 COPY prepare.sh /usr/bin
 COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json
@@ -36,36 +35,40 @@ RUN sudo chown -R $NB_USER /gcs
 
 
 ## more apt-get
-RUN sudo apt-get install -yq \
+RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
   vim less man locate kmod dialog \
-  octave && \
+  octave octave-optim gnuplot fonts-freefont-otf ghostscript && \
   sudo apt-get clean
 
 
 # set up conda channels
 RUN mkdir /opt/conda/specs
-COPY base_environment.yml /opt/conda/specs
+COPY shared_resources/base_environment.yml /opt/conda/specs
+COPY shared_resources/scheduler_environment.yml /opt/conda/specs
 COPY notebook_environment.yml /opt/conda/specs
 COPY r_environment.yml /opt/conda/specs
-RUN conda config --add channels conda-forge/label/dev && \
-  conda config --add channels conda-forge
-
+RUN conda config --add channels conda-forge && \
+  conda config --set channel_priority strict
 
 ## set up conda
 RUN conda update -n base conda
 
 #  update environemnt with common packages across worker and nb
-RUN conda env update -f /opt/conda/specs/base_environment.yml
+RUN conda env update -f /opt/conda/specs/scheduler_environment.yml && \
+  conda env update -f /opt/conda/specs/base_environment.yml
+
+# need access to settings/page_config.json for elyra
+USER root
+RUN /tempdir/fix-permissions.sh /opt/conda/share/jupyter/lab/settings
+USER $NB_USER
 
 # update environment with nb-specific packages
 RUN conda env update -f /opt/conda/specs/notebook_environment.yml
 
-RUN conda list -n base
-
-# create r environment
+# add r env
 RUN conda env create -f /opt/conda/specs/r_environment.yml
 
-RUN conda list -n r
+RUN conda list -n base
 
 ## Set up jupyter lab extensions
 RUN jupyter labextension update --all && \
@@ -78,15 +81,23 @@ RUN jupyter labextension update --all && \
     jupyter-leaflet \
     jupyter-matplotlib \
     jupyterlab-plotly \
+    jupyterlab-tabular-data-editor \
     plotlywidget
+RUN jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix
+
+## configure nbdime
+RUN source /opt/conda/etc/profile.d/conda.sh \
+  && conda activate \
+  && nbdime config-git --enable --system
+# makes sure that the web-tools allow accessible IP
+COPY nbdime_config.json /opt/conda/etc/jupyter
 
 
 ## clean up
 RUN sudo rm -rf /var/lib/apt/lists/* /tempdir
-RUN conda clean --all -f
-
+RUN conda clean -yaf
 
 ## prepare container
 WORKDIR $HOME
 ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
-CMD ["start.sh jupyter lab"]
+CMD ["start-notebook.sh"]
diff --git a/notebook/add_service_creds.py b/notebook/add_service_creds.py
deleted file mode 100644
index 44163e1..0000000
--- a/notebook/add_service_creds.py
+++ /dev/null
@@ -1,38 +0,0 @@
-import json, yaml, os, glob
-
-
-def add_service_creds():
-
-    with open('/home/jovyan/worker-template.yml', 'r') as f:
-        WORKER_TEMPLATE = yaml.safe_load(f)
-        
-    env_vars = []
-    creds = {}
-
-    for env in WORKER_TEMPLATE['spec']['containers'][0]['env']:
-        if 'GCSFUSE_TOKEN' in env['name']:
-            continue
-        elif 'GCSFUSE_TOKENS' in env['name']:
-            creds.update(env['value'])
-        else:
-            env_vars.append(env)
-
-    for f in glob.glob('/home/jovyan/service-account-credentials/*.json'):
-        bucket = os.path.splitext(os.path.basename(f))[0]
-
-        with open(f, 'r') as f:
-            creds[bucket] = json.load(f)
-
-    env_vars.append(
-        {'name': 'GCSFUSE_TOKENS', 'value': json.dumps(creds)})
-    
-    WORKER_TEMPLATE['spec']['containers'][0]['env'] = env_vars
-
-    with open('/home/jovyan/worker-template.yml', 'w') as f:
-        f.write(yaml.dump(WORKER_TEMPLATE))
-
-    print('worker-template.yml updated')
-
-
-if __name__ == '__main__':
-    add_service_creds()
\ No newline at end of file
diff --git a/notebook/config.yaml b/notebook/config.yaml
old mode 100644
new mode 100755
index 3e8ec1a..34dee2a
--- a/notebook/config.yaml
+++ b/notebook/config.yaml
@@ -17,16 +17,10 @@ logging:
   tornado.application: error
 
 # the below section gives defaults for launching clusters from the dask
-# labextension viewer. We could customize this but right now it should load
-# a default cluster from the worker-template.yml file (as sepecified in the
-# following section)
+# labextension viewer. We could customize this further
 labextension:
   factory:
-    module: dask_kubernetes
-    class: KubeCluster
+    module: dask-gateway
+    class: GatewayCluster
     args: []
     kwargs: {}
-
-kubernetes:
-  worker-template-path: "/home/{NB_USER}/worker-template.yml"
-  name: "dask-{JUPYTERHUB_USER}-{uuid}"
diff --git a/notebook/nbdime_config.json b/notebook/nbdime_config.json
new file mode 100644
index 0000000..6bbdaca
--- /dev/null
+++ b/notebook/nbdime_config.json
@@ -0,0 +1,5 @@
+{
+    "WebTool": {
+        "ip": "*"
+    }
+}
\ No newline at end of file
diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml
old mode 100644
new mode 100755
index 81d85f6..9d9d810
--- a/notebook/notebook_environment.yml
+++ b/notebook/notebook_environment.yml
@@ -2,26 +2,36 @@ name: base
 channels:
   - conda-forge
 dependencies:
-  - black=19.10b0=py37_0
-  - coverage=4.5.4=py37h516909a_0
-  - dask-labextension=1.0.3=py_0
-  - flake8=3.7.9=py37_0
-  - ipdb=0.12.3=py_0
-  - ipyleaflet=0.11.6=py37_0
-  - ipympl=0.3.3=py_0
-  - jupyterlab_code_formatter=0.7.0=py_0
-  - nano=2.9.8=hb256ff8_1000
-  - nb_conda_kernels=2.2.2=py37_0
-  - nose=1.3.7=py37_1003
+  - black
+  - bump2version
+  - coverage
+  - dask-labextension
+  - flake8
+  - ipdb
+  - ipyleaflet
+  - ipympl
+  - ipypublish
+  - isort
+# need jupyterhub version to match that running on the hub
+  - jupyterhub=1.2.1 # pinkeep: jupyterhub=1.2.1
+  - jupyterlab
+  - jupyterlab-git
+  - jupyterlab_code_formatter
+  - jupyterlab-git
+  - mypy
+  - nano
+  - nbdime
+  - nb_conda_kernels
   - octave_kernel=0.31.0=py_0
   - oct2py=5.0.4=py_0
-  - openssh=7.9p1=h0fa992c_1
-  - pip=19.3.1=py37_0
-  - pytest=5.3.1=py37_0
-  - pytest-cov=2.8.1=py_0
-  - pytest-runner=5.2=py_0
-  - python-graphviz=0.13.2=py_0
-  - sphinx=2.2.2=py_0
-  - tox=3.14.2=py_0
+  - openssh
+  - papermill
+  - pip
+  - pytest
+  - pytest-cov
+  - python-graphviz
+  - sidecar
+  - sphinx_rtd_theme
+  - tox
   - pip:
     - black_nbconvert
diff --git a/notebook/overrides.json b/notebook/overrides.json
old mode 100644
new mode 100755
index 78dcf65..38e8912
--- a/notebook/overrides.json
+++ b/notebook/overrides.json
@@ -1,8 +1,14 @@
 {
   "@jupyterlab/fileeditor-extension:plugin": {
     "editorConfig": {
-        "rulers": [79],
+        "rulers": [88],
         "lineNumbers": true
     }
+  },
+  "@jupyterlab/notebook-extension:tracker": {
+    "codeCellConfig": {
+      "rulers": [88],
+      "lineNumbers": true
+    }
   }
 }
diff --git a/notebook/prepare.sh b/notebook/prepare.sh
old mode 100644
new mode 100755
index 8d1c835..6b38c04
--- a/notebook/prepare.sh
+++ b/notebook/prepare.sh
@@ -2,16 +2,14 @@
 
 set -x
 
-echo "Copy Dask configuration files from pre-load directory into home/.config"
-mkdir -p /home/jovyan/.config/dask
-cp --update -r -v /pre-home/config.yaml /home/jovyan/.config/dask/
+echo "Copy Dask configuration files from pre-load directory into opt/conda/etc/dask/"
+mkdir -p /opt/conda/etc/dask
+cp --update -r -v /pre-home/config.yaml /opt/conda/etc/dask/
 
-# should probably pick one of these!!! The second is new, but is implied by the
-# cp /pre-home below, and we actually only read the version in ~ in rhg_compute_tools.
-cp -r -v /pre-home/worker-template.yml /home/jovyan/.config/dask/
-cp -r -v /pre-home/worker-template.yml /home/jovyan/
+# set credentials for use when starting workers with dask-gateway
+python /pre-home/set_gateway_opts.py
 
-sudo rm /pre-home/config.yaml
+sudo rm /pre-home/config.yaml /pre-home/set_gateway_opts.py
 
 echo "Copy files from pre-load directory into home"
 cp --update -r -v /pre-home/. /home/jovyan
@@ -30,14 +28,15 @@ do
             echo "Mounting $bucket to /gcs/${bucket}";
             mkdir -p "/gcs/$bucket";
             /usr/bin/gcsfuse --key-file="$f" "$bucket" "/gcs/${bucket}";
+            echo "Including $bucket in dask-gateway options";
         fi;
     fi;
 done
 
-if [ -f "/home/jovyan/worker-template.yml" ]; then
-    echo "appending service-account-credentials to worker-template";
-    python /home/jovyan/add_service_creds.py;
-fi
+
+
+# needed for CLAWPACK to not throw segfaults sometimes
+ulimit -s unlimited
 
 # Run extra commands
 $@
diff --git a/notebook/run_sql_proxy.py b/notebook/run_sql_proxy.py
deleted file mode 100644
index 7450e01..0000000
--- a/notebook/run_sql_proxy.py
+++ /dev/null
@@ -1,168 +0,0 @@
-'''
-Run an SQL proxy server to enable connections to a cloud sql instance
-
-To use, create a file at /home/jovyan/setup.cfg with the following contents:
-
-.. code-block:: bash
-
-    [sql-proxy]
-
-    SQL_INSTANCE = {project}:{region}:{instance}=tcp:{port}
-    SQL_TOKEN_FILE = /path/to/credentials-file.json
-
-modifying the `SQL_INSTANCE` and `SQL_TOKEN` values to match your server's
-configuration.
-
-Then, run `python run_sql_proxy.py`. This will start an SQL proxy and will also
-add these credentials to your ~/worker-template.yml file.
-
-When the process is killed (through `^C` or by killing the process) the worker
-template will be returned to its previous state.
-
-'''
-
-import os
-import json
-import yaml
-import configparser
-import subprocess
-import signal
-import time
-
-
-def get_sql_service_account_token(sql_token_file):
-    if sql_token_file is None:
-        return
-
-    try:
-        with open(sql_token_file, 'r') as f:
-            return json.load(f)
-
-    except (OSError, IOError):
-        return
-
-
-class add_sql_proxy_to_worker_spec(object):
-    kill_now = False
-
-    def __init__(self, sql_instance, sql_token):
-        self.original_worker_template = None
-        self.sql_instance = sql_instance
-        self.sql_token = sql_token
-
-        self.sql_proxy_process = None
-        
-        # handle sigint
-        signal.signal(signal.SIGINT, self.return_worker_spec_to_original_state)
-        signal.signal(signal.SIGTERM, self.return_worker_spec_to_original_state)
-
-    def __enter__(self):
-        sql_instance = self.sql_instance
-        sql_token = self.sql_token
-
-        if (sql_instance is None) or (sql_token is None):
-            return
-
-        try:
-            with open('/home/jovyan/worker-template.yml', 'r') as f:
-                self.original_worker_template = f.read()
-                worker_template_modified = yaml.safe_load(self.original_worker_template)
-        
-        except (OSError, IOError, ValueError):
-            return
-            
-        env_vars = []
-
-        for env in worker_template_modified['spec']['containers'][0]['env']:
-            if 'SQL_INSTANCE' in env['name']:
-                continue
-            elif 'SQL_TOKEN' in env['name']:
-                continue
-            else:
-                env_vars.append(env)
-
-        env_vars.append(
-            {'name': 'SQL_TOKEN', 'value': json.dumps(sql_token)})
-
-        env_vars.append(
-            {'name': 'SQL_INSTANCE', 'value': sql_instance})
-        
-        worker_template_modified['spec']['containers'][0]['env'] = env_vars
-
-        with open('/home/jovyan/worker-template.yml', 'w') as f:
-            f.write(yaml.dump(worker_template_modified))
-
-        print('proxy added to worker-template.yml')
-
-
-    def maybe_start_sql_proxy(self, sql_instance, sql_token_file):
-        if (sql_instance is None) or (sql_token_file is None):
-            return
-        
-        p = subprocess.Popen([
-            '/usr/bin/cloud_sql_proxy',
-            '-instances',
-            sql_instance,
-            '-credential_file',
-            sql_token_file])
-
-        self.sql_proxy_process = p
-
-        p.wait()
-
-
-    def return_worker_spec_to_original_state(self, *args):
-        if self.original_worker_template is None:
-            return
-
-        with open('/home/jovyan/worker-template.yml', 'w') as f:
-            f.write(self.original_worker_template)        
-
-        print('proxy removed from worker-template.yml')
-
-        if (
-                (self.sql_proxy_process is not None)
-                and (self.sql_proxy_process.poll() is None)):
-
-            try:
-                self.sql_proxy_process.kill()
-            except:
-                pass
-        
-        self.kill_now = True
-
-
-    def __exit__(self, *errs):
-        self.return_worker_spec_to_original_state()
-
-
-def handle_sql_config():
-    config = configparser.ConfigParser()
-
-    if not os.path.isfile('/home/jovyan/setup.cfg'):
-        return
-
-    config.read('/home/jovyan/setup.cfg')
-    
-    if not 'sql-proxy' in config.sections():
-        return
-
-    sql_instance = config['sql-proxy'].get('SQL_INSTANCE')
-    sql_token_file = config['sql-proxy'].get('SQL_TOKEN_FILE')
-    sql_token = get_sql_service_account_token(sql_token_file)
-
-    sql_proxy = add_sql_proxy_to_worker_spec(sql_instance, sql_token)
-    
-    with sql_proxy:
-        sql_proxy.maybe_start_sql_proxy(sql_instance, sql_token_file)
-
-    # wait for sql_proxy to exit gracefully
-    while True:
-        if sql_proxy.kill_now:
-            break
-
-        time.sleep(1)
-
-
-if __name__ == "__main__":
-    handle_sql_config()
diff --git a/notebook/set_gateway_opts.py b/notebook/set_gateway_opts.py
new file mode 100644
index 0000000..d48a7d7
--- /dev/null
+++ b/notebook/set_gateway_opts.py
@@ -0,0 +1,36 @@
+from pathlib import Path
+import json
+import yaml
+import os
+
+cred_files = Path("/home/jovyan/service-account-credentials").glob("*.json")
+
+out_file = Path("/opt/conda/etc/dask/gateway.yaml")
+out_file.parent.mkdir(exist_ok=True, parents=True)
+
+# get tokens
+tokens = {}
+for fpath in cred_files:
+    bucket = fpath.stem
+    with open(fpath, "r") as file:
+        tokens[bucket] = json.load(file)
+
+# get image names
+scheduler_image = os.environ["JUPYTER_IMAGE_SPEC"].replace("/notebook:","/scheduler:")
+worker_image = os.environ["JUPYTER_IMAGE_SPEC"].replace("/notebook:","/worker:")
+
+# create config dict
+config = {
+    "gateway": {
+        "cluster": {
+            "options": {
+                "gcsfuse_tokens": json.dumps(tokens).replace("{","{{").replace("}","}}"),
+                "scheduler_image": scheduler_image,
+                "worker_image": worker_image
+            }
+        }
+    }
+}
+
+with open(out_file, "w") as fout:
+    yaml.safe_dump(config, fout)
\ No newline at end of file
diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml
deleted file mode 100644
index a8990c2..0000000
--- a/notebook/worker-template.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-metadata:
-spec:
-  restartPolicy: Never
-  containers:
-  - args:
-    - dask-worker
-    - --nthreads
-    - '1'
-    - --no-dashboard
-    - --memory-limit
-    - 11.5GB
-    - --death-timeout
-    - '60'
-    env:
-      - name: GCSFUSE_BUCKET
-        value: rhg-data
-    image: rhodium/worker:WILL_BE_REPLACED_BY_TRAVIS
-    name: dask-worker
-    securityContext:
-      capabilities:
-        add: [SYS_ADMIN]
-      privileged: true
-    resources:
-      limits:
-        cpu: "1.75"
-        memory: 11.5G
-      requests:
-        cpu: "1.75"
-        memory: 11.5G
diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile
index 75039f0..af386d1 100644
--- a/octave-worker/Dockerfile
+++ b/octave-worker/Dockerfile
@@ -2,11 +2,18 @@ ARG TRAVIS_COMMIT=${TRAVIS_COMMIT}
 FROM rhodium/worker:${TRAVIS_COMMIT}
 ARG DEBIAN_FRONTEND=noninteractive
 
+COPY shared_resources /tempdir
+
 ## filepath curation
 COPY octave_environment.yml /opt/conda/specs/octave_environment.yml
 
 ## install octave
-RUN sudo apt-get update && sudo apt-get install -yq octave
+RUN apt-get update \
+    && apt-get install -yq --no-install-recommends octave octave-optim
 
 # add octave-specific packages
 RUN conda env update -f /opt/conda/specs/octave_environment.yml
+
+RUN rm -rf /tempdir \
+    && conda clean -yaf \
+    && sudo apt-get clean
diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml
old mode 100644
new mode 100755
diff --git a/pin.py b/pin.py
index 175ab22..37ce9eb 100644
--- a/pin.py
+++ b/pin.py
@@ -25,6 +25,15 @@
 from ruamel.yaml import YAML
 
 
+SPEC_FILES = [
+    ('shared_resources/base_environment.yml', 'base'),
+    ('shared_resources/scheduler_environment.yml', 'base'),
+    ('notebook/notebook_environment.yml', 'base'),
+    ('octave-worker/octave_environment.yml', 'base'),
+    ('notebook/r_environment.yml', 'r'),
+]
+
+
 def get_versions_in_current_environment(envname='base'):
     '''
     Calls ``conda env export -n {envname} --json`` and returns spec
@@ -294,26 +303,18 @@ def pinversions():
 def pin(file, dry_run):
     '''Pin packages in environment files based on environments on the local machine'''
     
-    spec_files = [
-            ('base_environment.yml', 'base'),
-            ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base'),
-            ('notebook/r_environment.yml', 'r')]
-    
     if file == 'all':
-        pin_files(spec_files, dry_run=dry_run)
+        pin_files(SPEC_FILES, dry_run=dry_run)
     elif file == 'base':
-        pin_files([spec_files[0]], dry_run=dry_run)
+        pin_files([SPEC_FILES[0]], dry_run=dry_run)
     elif file == 'notebook':
-        pin_files([spec_files[1]], dry_run=dry_run)
+        pin_files([SPEC_FILES[1]], dry_run=dry_run)
     elif file == 'octave':
-        pin_files([spec_files[2]], dry_run=dry_run)
-    elif file == 'r':
-        pin_files([spec_files[3]], dry_run=dry_run)
+        pin_files([SPEC_FILES[2]], dry_run=dry_run)
     else:
         raise ValueError(
             'env type not recognized: {}'
-            'choose from "base", "notebook", "octave", "r", or "all".'
+            'choose from "base", "notebook", "octave", or "all".'
             .format(file))
 
         
@@ -329,26 +330,22 @@ def pin(file, dry_run):
 def unpin(file, dry_run):
     '''Unpin packages in environment files'''
     
-    spec_files = [
-            ('base_environment.yml', 'base'),
-            ('notebook/notebook_environment.yml', 'base'),
-            ('octave-worker/octave_environment.yml', 'base'),
-            ('notebook/r_environment.yml', 'r')]
-    
     if file == 'all':
-        unpin_files(spec_files, dry_run=dry_run)
+        unpin_files(SPEC_FILES, dry_run=dry_run)
     elif file == 'base':
-        unpin_files([spec_files[0]], dry_run=dry_run)
+        unpin_files([SPEC_FILES[0]], dry_run=dry_run)
+    elif file == 'scheduler':
+        unpin_files([SPEC_FILES[1]], dry_run=dry_run)
     elif file == 'notebook':
-        unpin_files([spec_files[1]], dry_run=dry_run)
+        unpin_files([SPEC_FILES[2]], dry_run=dry_run)
     elif file == 'octave':
-        unpin_files([spec_files[2]], dry_run=dry_run)
+        unpin_files([SPEC_FILES[3]], dry_run=dry_run)
     elif file == 'r':
-        unpin_files([spec_files[3]], dry_run=dry_run)
+        unpin_files([SPEC_FILES[4]], dry_run=dry_run)
     else:
         raise ValueError(
             'env type not recognized: {}'
-            'choose from "base", "notebook", "octave", "r", or "all".'
+            'choose from "base", "scheduler", "notebook", "octave", "r", or "all".'
             .format(file))
 
 
diff --git a/base_environment.yml b/shared_resources/base_environment.yml
old mode 100644
new mode 100755
similarity index 73%
rename from base_environment.yml
rename to shared_resources/base_environment.yml
index ea54de9..990c316
--- a/base_environment.yml
+++ b/shared_resources/base_environment.yml
@@ -11,13 +11,11 @@ dependencies:
   - cartopy=0.17.0=py37h423102d_1009
   - cftime=1.0.4.2=py37hc1659b7_0
   - click=7.0=py_0
-  - compilers=1.0.4=0
-  - dask=2.8.1=py_0
+  - compilers=1.1.1=0
   - dask-glm=0.2.0=py_1
   - dask-ml=1.1.1=py_0
   - datashader=0.8.0=py_0
   - descartes=1.1.0=py_4
-  - distributed=2.8.1=py_0
   - dropbox=9.4.0=py_0
 # need to make sure we get esmpy compiled with mpi otherwise xesmf regridding
 # won't work
@@ -25,11 +23,6 @@ dependencies:
   - fastparquet=0.3.2=py37hc1659b7_0
   - fiona=1.8.13=py37h900e953_0
   - fusepy=3.0.1=py_0
-# this gcc pin is necessary b/c of a weird feature in the h553295d_15 build
-# which makes it hard to build numpy-based cython extensions (like pyclaw).
-# we should try removing it whenever we next do an update and see if Clawpack
-# can still be built
-  - gcc_linux-64=7.3.0=h553295d_14 # pinkeep: gcc_linux-64=7.3.0=h553295d_14
   - gcsfs=0.5.3=py_0
   - gdal=3.0.2=py37hbb6b9fb_5
   - geoalchemy2=0.6.3=py_0
@@ -38,19 +31,17 @@ dependencies:
   - geotiff=1.5.1=hbd99317_7
   - geoviews=1.6.6=py_0
   - git=2.24.0=pl526hce37bd2_1
-  - gitpython=3.0.5=py_0
+  - gitpython=3.1.8=py_0
   - google-cloud-container=0.3.0=py37_0
   - google-cloud-storage=1.23.0=py37_0
   - holoviews=1.12.7=py_0
   - h5netcdf=0.7.4=py_0
   - icu=64.2=he1b5a44_1
+  - intake-esm=2020.6.11
   - iris=2.2.0=py37_1003
   - jedi=0.15.1=py37_0
-# need server proxy on workers if using remote scheduler
-  - jupyter-server-proxy=1.3.2=py_0
   - kubernetes
   - lapack=3.6.1=ha44fe06_2
-  - lz4=2.2.1=py37hd79334b_0
   - make=4.2.1=h14c3975_2004
   - matplotlib=3.1.2=py37_1
   - nc-time-axis=1.2.0=py_0
@@ -59,12 +50,12 @@ dependencies:
   - netcdf4=1.5.3=mpi_mpich_py37h01ee55b_1
   - numba=0.46.0=py37hb3f55d8_1
   - numcodecs=0.6.4=py37he1b5a44_0
-  - pandas=0.25.3=py37hb3f55d8_0
   # for geoviews
   - phantomjs=2.1.1=1
   - pip=19.3.1=py37_0
   - plotly=4.3.0=py_0
   - polyline=1.4.0=py_0
+  - pydap=3.2.2=py37_1000
   - pygeos=0.5=py37h5d51c17_1
   - pyinterp=0.0.7=py37h97f2665_0
   - pyshp=2.1.0=py_0
@@ -85,10 +76,9 @@ dependencies:
   - shapely=1.6.4=py37h5d51c17_1007
   - sparse=0.8.0=py_0
   - statsmodels=0.10.2=py37hc1659b7_0
-  - tini=0.18.0=h14c3975_1001
   - unzip=6.0=h516909a_0
   - uritemplate=3.0.0=py_1
-  - xarray=0.14.1=py_0
+  - xarray=0.16.0=py_0
   - xesmf=0.2.1=py_0
   - xgcm=0.2.0=py_0
   - xhistogram=0.1.1=py_0
@@ -104,9 +94,4 @@ dependencies:
     - climate-toolbox==0.1.5
     - impactlab-tools==0.4.0
     - parameterize-jobs==0.1.1
-    - rhg_compute_tools==0.2.2
-    - git+https://github.com/NCAR/intake-esm.git@v2020.3.16.2#egg=intake_esm
-# need to install from master until 0.10.1
-# due to handling of remote scheduler
-# (we also should at some point switch to dask-gateway instead of dask-kubernetes)
-    - dask_kubernetes==0.10.1
+    - git+https://github.com/RhodiumGroup/rhg_compute_tools.git@gateway#egg=rhg_compute_tools # needed for dask-gateway until merged into master
\ No newline at end of file
diff --git a/common.sh b/shared_resources/common.sh
old mode 100644
new mode 100755
similarity index 84%
rename from common.sh
rename to shared_resources/common.sh
index 3078d6e..21e2eb8
--- a/common.sh
+++ b/shared_resources/common.sh
@@ -5,8 +5,10 @@ apt-get install -yq --no-install-recommends \
   apt-utils \
   zip \
   bzip2 \
+  unzip \
   ca-certificates \
   curl \
+  locales \
   lsb-release \
   gnupg2 \
   sudo \
@@ -38,8 +40,3 @@ chmod +x /usr/bin/cloud_sql_proxy
 chmod +x /usr/bin/prepare.sh
 mkdir /gcs
 mkdir /opt/app
-
-# super sketchy hack to get around our need for compiler_compat binaries and some
-# other things that cause problems together?
-# see https://github.com/ContinuumIO/anaconda-issues/issues/11152
-rm -rf /opt/conda/compiler_compat/ld
diff --git a/shared_resources/fix-permissions.sh b/shared_resources/fix-permissions.sh
new file mode 100755
index 0000000..c8d0032
--- /dev/null
+++ b/shared_resources/fix-permissions.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+## Adapted from:
+## https://github.com/jupyter/docker-stacks/blob/master/base-notebook/fix-permissions
+
+# set permissions on a directory
+# after any installation, if a directory needs to be (human) user-writable,
+# run this script on it.
+# It will make everything in the directory owned by the group $NB_GID
+# and writable by that group.
+# Deployments that want to set a specific user id can preserve permissions
+# by adding the `--group-add users` line to `docker run`.
+
+# uses find to avoid touching files that already have the right permissions,
+# which would cause massive image explosion
+
+# right permissions are:
+# group=$NB_GID
+# AND permissions include group rwX (directory-execute)
+# AND directories have setuid,setgid bits set
+
+set -e
+
+for d in "$@"; do
+  find "$d" \
+    ! \( \
+      -group $NB_GID \
+      -a -perm -g+rwX  \
+    \) \
+    -exec chgrp $NB_GID {} \; \
+    -exec chmod g+rwX {} \;
+  # setuid,setgid *on directories only*
+  find "$d" \
+    \( \
+        -type d \
+        -a ! -perm -6000  \
+    \) \
+    -exec chmod +6000 {} \;
+done
\ No newline at end of file
diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml
new file mode 100644
index 0000000..f08e3bc
--- /dev/null
+++ b/shared_resources/scheduler_environment.yml
@@ -0,0 +1,15 @@
+# Only the packages necessary to build a dask-gateway scheduler. As seen here:
+# https://github.com/dask/dask-gateway/blob/master/dask-gateway/Dockerfile
+channels:
+  - conda-forge
+dependencies:
+  - aiohttp=3.7.3=py38h25fe258_0
+  - dask=2.30.0=py_0
+  - distributed=2.30.1=py38h578d9bd_0
+  - dask-gateway=0.9.0=py38h578d9bd_0
+  - lz4=3.1.1=py38h87b837d_0
+  - jupyter-server-proxy=1.5.0=py_0
+  - numpy=1.19.4=py38hf0fd68c_1
+  - pandas=1.1.4=py38h0ef3d22_0
+  - tini=0.18.0=h14c3975_1001
+name: base
\ No newline at end of file
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 13ae443..8981dba 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -1,56 +1,14 @@
-## using same container as jupyter/base-notebook:python-3.7.3
-FROM ubuntu:bionic-20190612@sha256:6e9f67fa63b0323e9a1e587fd71c561ba48a034504fb804fd26fd8800039835d
+## using same base image as the "base image for the base image" of the notebook image
+FROM rhodium/scheduler:local
 ARG DEBIAN_FRONTEND=noninteractive
-ENV NPY_DISTUTILS_APPEND_FLAGS=1
-
-## filepath curation
-RUN mkdir /tempdir
-COPY common.sh /tempdir
-
-COPY add_service_creds.py /usr/bin
-COPY prepare.sh /usr/bin
-
 
 ## perform a bunch of common actions
 RUN bash /tempdir/common.sh
 
-
-###########
-## install miniconda
-# adapted from continuumio/miniconda3
-
-ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
-ENV PATH /opt/conda/bin:$PATH
-
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh -O ~/miniconda.sh && \
-    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
-    rm ~/miniconda.sh && \
-    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-    echo "conda activate base" >> ~/.bashrc
-###########
-
-
-## set up conda channels
-RUN mkdir /opt/conda/specs
-COPY base_environment.yml /opt/conda/specs
-RUN conda config --add channels conda-forge/label/dev && \
-  conda config --add channels conda-forge
-
-
-## set up python env
-RUN conda update -n base conda
 RUN conda env update -f /opt/conda/specs/base_environment.yml
 RUN conda list -n base
 
 ## clean up
-RUN rm -rf /var/lib/apt/lists/* /tempdir
-RUN conda clean --all -f
-
-
-## prepare container
-ENV OMP_NUM_THREADS=1
-ENV MKL_NUM_THREADS=1
-ENV OPENBLAS_NUM_THREADS=1
-
-ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
+RUN rm -rf /var/lib/apt/lists/* /tempdir \
+    && conda clean -yaf \
+    && sudo apt-get clean
\ No newline at end of file
diff --git a/worker/Dockerfile_scheduler b/worker/Dockerfile_scheduler
new file mode 100644
index 0000000..2a0030c
--- /dev/null
+++ b/worker/Dockerfile_scheduler
@@ -0,0 +1,72 @@
+## using same base image as the "base image for the base image" of the notebook image
+FROM ubuntu:bionic-20190612@sha256:6e9f67fa63b0323e9a1e587fd71c561ba48a034504fb804fd26fd8800039835d
+ARG DEBIAN_FRONTEND=noninteractive
+
+# needed to properly install packages that use numpy libraries
+ENV NPY_DISTUTILS_APPEND_FLAGS=1
+
+# needed so that matplotlib will work headless
+ENV MPLBACKEND=Agg
+
+## filepath curation
+COPY shared_resources /tempdir
+COPY add_service_creds.py /usr/bin
+COPY prepare.sh /usr/bin
+
+# install apt-get packages
+RUN apt-get update -y --no-install-recommends
+RUN apt-get install -yq --no-install-recommends \
+  ca-certificates \
+  locales \
+  sudo \
+  wget
+
+
+###########
+## install miniconda (following jupyter/base-notebook
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    locale-gen
+    
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV CONDA_DIR=/opt/conda \
+    SHELL=/bin/bash \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
+ENV PATH=$CONDA_DIR/bin:$PATH
+
+ENV MINICONDA_VERSION=4.7.12.1 \
+    MINICONDA_MD5=81c773ff87af5cfac79ab862942ab6b3
+
+WORKDIR /tmp
+RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    echo "${MINICONDA_MD5} *Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
+    /bin/bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    conda config --system --prepend channels conda-forge && \
+    conda config --system --set channel_priority strict && \
+    conda update -n base --yes conda && \
+    conda update --all --yes
+###########
+
+## set up python env
+RUN mkdir /opt/conda/specs
+COPY shared_resources/scheduler_environment.yml /opt/conda/specs
+COPY shared_resources/base_environment.yml /opt/conda/specs
+RUN conda env update -f /opt/conda/specs/scheduler_environment.yml
+
+## clean up
+RUN rm -rf /var/lib/apt/lists/* /tmp/* \
+    && conda clean -yaf \
+    && sudo apt-get clean
+
+## prepare container
+ENV OMP_NUM_THREADS=1
+ENV MKL_NUM_THREADS=1
+ENV OPENBLAS_NUM_THREADS=1
+
+WORKDIR /
+
+RUN chmod +x /usr/bin/prepare.sh
+
+ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"]
\ No newline at end of file