From 7a7dc8cfabc8b2154053596352d73ba28f8224c0 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 28 Sep 2018 13:01:40 -0700 Subject: [PATCH 1/4] build google-cloud-sdk on notebook & worker --- notebook/Dockerfile | 13 ++++++++----- worker/Dockerfile | 21 ++++++++++++++++++--- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 211bf68..13fe75a 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,5 +1,3 @@ -# Note that this image is attempting to update from the build here: https://travis-ci.org/RhodiumGroup/docker_images/builds/400944547#L2345 - ##We build our single-user notebook from the base jupyter notebook image FROM jupyter/base-notebook @@ -19,9 +17,12 @@ RUN apt-get update \ libcurl4-openssl-dev \ libfuse-dev \ libssl-dev \ + locate \ + lsb-release \ nano \ rsync \ tzdata \ + unzip \ vim \ zip @@ -121,9 +122,7 @@ RUN pip install \ coverage==4.5.1 \ dask-kubernetes==0.4.0 \ flake8==3.5.0 \ - google-cloud-storage==1.10.0 \ - google-cloud==0.32.0 \ - gsutil==4.33 \ + google-cloud-storage==1.12.0 \ impactlab-tools \ jupyterlab==0.32.1 \ kubernetes==6.0.0 \ @@ -160,6 +159,10 @@ COPY run_sql_proxy.py /pre-home COPY config.yaml /pre-home COPY worker-template.yml /pre-home +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update -y && apt-get install google-cloud-sdk -y RUN export GCSFUSE_REPO=gcsfuse-xenial \ && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ diff --git a/worker/Dockerfile b/worker/Dockerfile index 46c3704..eea2a62 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -7,11 +7,23 @@ RUN chmod +x /usr/local/bin/dumb-init USER root RUN apt-get update \ && apt-get install -yq --no-install-recommends \ + build-essential \ fuse \ + gfortran \ + git \ gnupg \ gnupg2 \ + keychain \ + libcurl4-openssl-dev \ libfuse-dev \ + libssl-dev \ + locate \ + lsb-release \ nano \ + rsync \ + tzdata \ + unzip \ + vim \ zip RUN conda update --yes conda @@ -78,6 +90,11 @@ RUN conda create -n worker --yes -c conda-forge \ RUN conda install -n worker --yes --channel conda-forge/label/dev geopandas +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update -y && apt-get install google-cloud-sdk -y + RUN export GCSFUSE_REPO=gcsfuse-xenial \ && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ @@ -87,9 +104,7 @@ RUN export GCSFUSE_REPO=gcsfuse-xenial \ RUN /opt/conda/envs/worker/bin/pip install \ dask-kubernetes==0.4.0 \ - google-cloud-storage==1.10.0 \ - google-cloud==0.32.0 \ - gsutil==4.33 \ + google-cloud-storage==1.12.0 \ pyasn1-modules==0.2.2 \ pyasn1==0.4.3 \ wget==3.2 \ From 8e0c79632b3351523f76701deebbdbfa2448fcbe Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 28 Sep 2018 14:29:35 -0700 Subject: [PATCH 2/4] set up worker to automatically initialize google cloud credentials with GCLOUD_DEFAULT_TOKEN_FILE --- worker/prepare.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/worker/prepare.sh b/worker/prepare.sh index df95e7e..23d470c 100644 --- a/worker/prepare.sh +++ b/worker/prepare.sh @@ -45,6 +45,9 @@ if [[ "$SQL_TOKEN" ]]; then fi; fi +if [[ "$GCLOUD_DEFAULT_TOKEN_FILE" ]]; then + gcloud auth activate-service-account --key-file $GCLOUD_DEFAULT_TOKEN_FILE; +fi # Run extra commands source activate worker && $@ From 59a8e937611d20ee1f5c49f302c7b57ff22535d0 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Sat, 29 Sep 2018 16:39:07 -0700 Subject: [PATCH 3/4] pin pip to 10.0.1 --- notebook/Dockerfile | 5 ++++- worker/Dockerfile | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 13fe75a..6564707 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -76,6 +76,7 @@ RUN conda install --yes -c conda-forge \ pandas=0.23.2 \ python-blosc=1.4.4 \ python-snappy=0.5.3 \ + pip=10.0.1 \ rasterio=0.36.0 \ scikit-image=0.14.0 \ scikit-learn=0.19.1 \ @@ -188,7 +189,9 @@ RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CO USER $NB_USER RUN pip install \ - rhg_compute_tools \ + rhg_compute_tools==0.1.0 \ + impactlab-tools==0.3.1 \ + climate-toolbox==0.1.4 \ --no-cache-dir ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] diff --git a/worker/Dockerfile b/worker/Dockerfile index eea2a62..3b5c781 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -72,6 +72,7 @@ RUN conda create -n worker --yes -c conda-forge \ numcodecs=0.5.5 \ numpy=1.14.2 \ pandas=0.23.2 \ + pip=10.0.1 \ python-blosc=1.4.4 \ python-snappy=0.5.3 \ rasterio=0.36.0 \ @@ -129,8 +130,10 @@ RUN chmod +x /usr/bin/prepare.sh RUN mkdir /opt/app RUN mkdir /gcs -RUN pip install \ - rhg_compute_tools \ +RUN /opt/conda/envs/worker/bin/pip install \ + rhg_compute_tools==0.1.0 \ + impactlab-tools==0.3.1 \ + climate-toolbox==0.1.4 \ --no-cache-dir ENTRYPOINT ["/usr/local/bin/dumb-init", "/usr/bin/prepare.sh"] \ No newline at end of file From 6b8eb690390fc320f680462b035717df8aa5e67e Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Sat, 29 Sep 2018 18:48:57 -0700 Subject: [PATCH 4/4] downgrade pip to 9.0.3 --- notebook/Dockerfile | 4 ++-- worker/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 6564707..6ecba93 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -76,7 +76,8 @@ RUN conda install --yes -c conda-forge \ pandas=0.23.2 \ python-blosc=1.4.4 \ python-snappy=0.5.3 \ - pip=10.0.1 \ + pip=9.0.3 \ + PyYAML=3.12 \ rasterio=0.36.0 \ scikit-image=0.14.0 \ scikit-learn=0.19.1 \ @@ -132,7 +133,6 @@ RUN pip install \ pytest-cov==2.5.1 \ pytest-runner==4.2 \ pytest==3.6.2 \ - PyYAML==3.12 \ Sphinx==1.7.5 \ tox==3.1.1 \ wget==3.2 \ diff --git a/worker/Dockerfile b/worker/Dockerfile index 3b5c781..b7f77b4 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -72,9 +72,10 @@ RUN conda create -n worker --yes -c conda-forge \ numcodecs=0.5.5 \ numpy=1.14.2 \ pandas=0.23.2 \ - pip=10.0.1 \ + pip=9.0.3 \ python-blosc=1.4.4 \ python-snappy=0.5.3 \ + PyYAML=3.12 \ rasterio=0.36.0 \ scikit-image=0.14.0 \ scikit-learn=0.19.1 \