diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 211bf68..6ecba93 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,5 +1,3 @@ -# Note that this image is attempting to update from the build here: https://travis-ci.org/RhodiumGroup/docker_images/builds/400944547#L2345 - ##We build our single-user notebook from the base jupyter notebook image FROM jupyter/base-notebook @@ -19,9 +17,12 @@ RUN apt-get update \ libcurl4-openssl-dev \ libfuse-dev \ libssl-dev \ + locate \ + lsb-release \ nano \ rsync \ tzdata \ + unzip \ vim \ zip @@ -75,6 +76,8 @@ RUN conda install --yes -c conda-forge \ pandas=0.23.2 \ python-blosc=1.4.4 \ python-snappy=0.5.3 \ + pip=9.0.3 \ + PyYAML=3.12 \ rasterio=0.36.0 \ scikit-image=0.14.0 \ scikit-learn=0.19.1 \ @@ -121,9 +124,7 @@ RUN pip install \ coverage==4.5.1 \ dask-kubernetes==0.4.0 \ flake8==3.5.0 \ - google-cloud-storage==1.10.0 \ - google-cloud==0.32.0 \ - gsutil==4.33 \ + google-cloud-storage==1.12.0 \ impactlab-tools \ jupyterlab==0.32.1 \ kubernetes==6.0.0 \ @@ -132,7 +133,6 @@ RUN pip install \ pytest-cov==2.5.1 \ pytest-runner==4.2 \ pytest==3.6.2 \ - PyYAML==3.12 \ Sphinx==1.7.5 \ tox==3.1.1 \ wget==3.2 \ @@ -160,6 +160,10 @@ COPY run_sql_proxy.py /pre-home COPY config.yaml /pre-home COPY worker-template.yml /pre-home +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update -y && apt-get install google-cloud-sdk -y RUN export GCSFUSE_REPO=gcsfuse-xenial \ && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ @@ -185,7 +189,9 @@ RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CO USER $NB_USER RUN pip install \ - rhg_compute_tools \ + rhg_compute_tools==0.1.0 \ + impactlab-tools==0.3.1 \ + climate-toolbox==0.1.4 \ --no-cache-dir ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] diff --git a/worker/Dockerfile b/worker/Dockerfile index 46c3704..b7f77b4 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -7,11 +7,23 @@ RUN chmod +x /usr/local/bin/dumb-init USER root RUN apt-get update \ && apt-get install -yq --no-install-recommends \ + build-essential \ fuse \ + gfortran \ + git \ gnupg \ gnupg2 \ + keychain \ + libcurl4-openssl-dev \ libfuse-dev \ + libssl-dev \ + locate \ + lsb-release \ nano \ + rsync \ + tzdata \ + unzip \ + vim \ zip RUN conda update --yes conda @@ -60,8 +72,10 @@ RUN conda create -n worker --yes -c conda-forge \ numcodecs=0.5.5 \ numpy=1.14.2 \ pandas=0.23.2 \ + pip=9.0.3 \ python-blosc=1.4.4 \ python-snappy=0.5.3 \ + PyYAML=3.12 \ rasterio=0.36.0 \ scikit-image=0.14.0 \ scikit-learn=0.19.1 \ @@ -78,6 +92,11 @@ RUN conda create -n worker --yes -c conda-forge \ RUN conda install -n worker --yes --channel conda-forge/label/dev geopandas +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update -y && apt-get install google-cloud-sdk -y + RUN export GCSFUSE_REPO=gcsfuse-xenial \ && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ @@ -87,9 +106,7 @@ RUN export GCSFUSE_REPO=gcsfuse-xenial \ RUN /opt/conda/envs/worker/bin/pip install \ dask-kubernetes==0.4.0 \ - google-cloud-storage==1.10.0 \ - google-cloud==0.32.0 \ - gsutil==4.33 \ + google-cloud-storage==1.12.0 \ pyasn1-modules==0.2.2 \ pyasn1==0.4.3 \ wget==3.2 \ @@ -114,8 +131,10 @@ RUN chmod +x /usr/bin/prepare.sh RUN mkdir /opt/app RUN mkdir /gcs -RUN pip install \ - rhg_compute_tools \ +RUN /opt/conda/envs/worker/bin/pip install \ + rhg_compute_tools==0.1.0 \ + impactlab-tools==0.3.1 \ + climate-toolbox==0.1.4 \ --no-cache-dir ENTRYPOINT ["/usr/local/bin/dumb-init", "/usr/bin/prepare.sh"] \ No newline at end of file diff --git a/worker/prepare.sh b/worker/prepare.sh index df95e7e..23d470c 100644 --- a/worker/prepare.sh +++ b/worker/prepare.sh @@ -45,6 +45,9 @@ if [[ "$SQL_TOKEN" ]]; then fi; fi +if [[ "$GCLOUD_DEFAULT_TOKEN_FILE" ]]; then + gcloud auth activate-service-account --key-file $GCLOUD_DEFAULT_TOKEN_FILE; +fi # Run extra commands source activate worker && $@