diff --git a/.gitignore b/.gitignore index 79e93af..15103b1 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ worker-config.yml .coverage __pycache__ .pytest_cache +.vscode/settings.json +.devcontainer.json +.DS_Store diff --git a/.travis.yml b/.travis.yml index 7032e46..ef07493 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,26 @@ -sudo: required - services: - docker env: -- IMAGE_NAME=notebook -- IMAGE_NAME=worker + jobs: + - IMAGE_NAME=notebook + - IMAGE_NAME=worker install: -- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak \ -'s/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' \ -notebook/worker-template.yml; else sed -i.bak \ -'s/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' \ -notebook/worker-template.yml; fi" -- "rm notebook/worker-template.yml.bak" -- "cat notebook/worker-template.yml | grep image:" -- "cp base_environment.yml $IMAGE_NAME/base_environment.yml" -- "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh" -- "cd $IMAGE_NAME" +- "cp -r shared_resources $IMAGE_NAME/shared_resources && chmod -R +x \ + $IMAGE_NAME/shared_resources" +- "if [[ $IMAGE_NAME == worker ]]; then cp -r shared_resources \ + octave-worker/shared_resources && chmod -R +x octave-worker/shared_resources; fi" +- cd $IMAGE_NAME script: -- docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT . +- "if [[ $IMAGE_NAME == worker ]]; then docker build -t \ + rhodium/scheduler:local -f Dockerfile_scheduler .; fi" +- travis_wait 90 docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT . +- "if [[ $IMAGE_NAME == worker ]]; then docker build -t \ + rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT \ + ../octave-worker; fi" deploy: - provider: script @@ -31,10 +30,9 @@ deploy: docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" - skip_cleanup: true on: all_branches: true - condition: $TRAVIS_BRANCH =~ ^dev + condition: $TRAVIS_BRANCH != master - provider: script script: >- @@ -44,7 +42,6 @@ deploy: docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" - skip_cleanup: true on: branch: master @@ -54,25 +51,51 @@ deploy: rhodium/$IMAGE_NAME:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" - skip_cleanup: true on: tags: true +# scheduler builds +- provider: script + script: >- + docker tag rhodium/scheduler:local rhodium/scheduler:$TRAVIS_COMMIT && + docker tag rhodium/scheduler:local rhodium/scheduler:$TRAVIS_BRANCH && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/scheduler:$TRAVIS_COMMIT" && + docker push "rhodium/scheduler:$TRAVIS_BRANCH" + on: + all_branches: true + condition: ($TRAVIS_BRANCH != master) && ($IMAGE_NAME == worker) + +- provider: script + script: >- + docker tag rhodium/scheduler:local rhodium/scheduler:latest && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/scheduler:$TRAVIS_COMMIT" && + docker push "rhodium/scheduler:latest" + on: + branch: master + condition: $IMAGE_NAME = worker + +- provider: script + script: >- + docker tag rhodium/scheduler:local rhodium/scheduler:$TRAVIS_TAG && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/scheduler:$TRAVIS_TAG" + on: + tags: true + condition: $IMAGE_NAME = worker + # octave-worker builds - provider: script script: >- - docker build -t rhodium/octave-worker:$TRAVIS_COMMIT - --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT ../octave-worker && docker tag rhodium/octave-worker:$TRAVIS_COMMIT rhodium/octave-worker:$TRAVIS_BRANCH && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && docker push "rhodium/octave-worker:$TRAVIS_BRANCH" - skip_cleanup: true on: all_branches: true - condition: $TRAVIS_BRANCH =~ ^dev - condition: $IMAGE_NAME = worker + condition: ($TRAVIS_BRANCH != master) && ($IMAGE_NAME == worker) - provider: script script: >- @@ -84,7 +107,6 @@ deploy: docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && docker push "rhodium/octave-worker:dev" && docker push "rhodium/octave-worker:latest" - skip_cleanup: true on: branch: master condition: $IMAGE_NAME = worker @@ -95,22 +117,6 @@ deploy: rhodium/octave-worker:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/octave-worker:$TRAVIS_TAG" - skip_cleanup: true on: tags: true - condition: $IMAGE_NAME = worker - - # - stage: alignment - # language: python - # python: - # - 3.6 - # script: - # - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - # - bash miniconda.sh -b -p $HOME/miniconda - # - export PATH="$HOME/miniconda/bin:$PATH" - # - hash -r - # - conda config --set always_yes yes --set changeps1 no - # - conda update -q conda - # - conda info -a - # - conda install pytest pytest-cov pyyaml - # - pytest + condition: $IMAGE_NAME = worker \ No newline at end of file diff --git a/bump.py b/bump.py index 42297a5..5a14384 100644 --- a/bump.py +++ b/bump.py @@ -5,7 +5,6 @@ SEARCH_PATTERNS = [ ('.travis.yml', r'(?P
.*TAG=)(?P\d{4}-\d{2}-\d{2})\.?(?P \d{2})?(?P [.\s]*)$'), - ('notebook/worker-template.yml', r'(?P .*image: rhodium/worker:)(?P\d{4}-\d{2}-\d{2})\.?(?P \d{2})?(?P [.\s]*)$'), ('jupyter-config.yml', r'(?P .*tag: )(?P\d{4}-\d{2}-\d{2})\.?(?P \d{2})?(?P [.\s]*)$')] diff --git a/notebook/Dockerfile b/notebook/Dockerfile old mode 100644 new mode 100755 index aad9f6d..ad4c39d --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,6 +1,9 @@ FROM jupyter/base-notebook:notebook-6.0.0 ARG DEBIAN_FRONTEND=noninteractive +# set shell to bash so that later can use source +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + ## needed to make sure things with cython compile correctly ## (this will eventually become default in numpy) ENV NPY_DISTUTILS_APPEND_FLAGS=1 @@ -18,13 +21,9 @@ USER $NB_USER ## filepath curation RUN sudo mkdir /pre-home && sudo chown -R $NB_USER /pre-home -COPY worker-template.yml /pre-home -COPY add_service_creds.py /pre-home -COPY run_sql_proxy.py /pre-home -COPY config.yaml /pre-home +COPY config.yaml set_gateway_opts.py /pre-home/ -RUN sudo mkdir /tempdir -COPY common.sh /tempdir +COPY shared_resources /tempdir COPY prepare.sh /usr/bin COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json @@ -36,36 +35,40 @@ RUN sudo chown -R $NB_USER /gcs ## more apt-get -RUN sudo apt-get install -yq \ +RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ vim less man locate kmod dialog \ - octave && \ + octave octave-optim gnuplot fonts-freefont-otf ghostscript && \ sudo apt-get clean # set up conda channels RUN mkdir /opt/conda/specs -COPY base_environment.yml /opt/conda/specs +COPY shared_resources/base_environment.yml /opt/conda/specs +COPY shared_resources/scheduler_environment.yml /opt/conda/specs COPY notebook_environment.yml /opt/conda/specs COPY r_environment.yml /opt/conda/specs -RUN conda config --add channels conda-forge/label/dev && \ - conda config --add channels conda-forge - +RUN conda config --add channels conda-forge && \ + conda config --set channel_priority strict ## set up conda RUN conda update -n base conda # update environemnt with common packages across worker and nb -RUN conda env update -f /opt/conda/specs/base_environment.yml +RUN conda env update -f /opt/conda/specs/scheduler_environment.yml && \ + conda env update -f /opt/conda/specs/base_environment.yml + +# need access to settings/page_config.json for elyra +USER root +RUN /tempdir/fix-permissions.sh /opt/conda/share/jupyter/lab/settings +USER $NB_USER # update environment with nb-specific packages RUN conda env update -f /opt/conda/specs/notebook_environment.yml -RUN conda list -n base - -# create r environment +# add r env RUN conda env create -f /opt/conda/specs/r_environment.yml -RUN conda list -n r +RUN conda list -n base ## Set up jupyter lab extensions RUN jupyter labextension update --all && \ @@ -78,15 +81,23 @@ RUN jupyter labextension update --all && \ jupyter-leaflet \ jupyter-matplotlib \ jupyterlab-plotly \ + jupyterlab-tabular-data-editor \ plotlywidget +RUN jupyter serverextension enable --py jupyterlab_code_formatter nbdime --sys-prefix + +## configure nbdime +RUN source /opt/conda/etc/profile.d/conda.sh \ + && conda activate \ + && nbdime config-git --enable --system +# makes sure that the web-tools allow accessible IP +COPY nbdime_config.json /opt/conda/etc/jupyter ## clean up RUN sudo rm -rf /var/lib/apt/lists/* /tempdir -RUN conda clean --all -f - +RUN conda clean -yaf ## prepare container WORKDIR $HOME ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] -CMD ["start.sh jupyter lab"] +CMD ["start-notebook.sh"] diff --git a/notebook/add_service_creds.py b/notebook/add_service_creds.py deleted file mode 100644 index 44163e1..0000000 --- a/notebook/add_service_creds.py +++ /dev/null @@ -1,38 +0,0 @@ -import json, yaml, os, glob - - -def add_service_creds(): - - with open('/home/jovyan/worker-template.yml', 'r') as f: - WORKER_TEMPLATE = yaml.safe_load(f) - - env_vars = [] - creds = {} - - for env in WORKER_TEMPLATE['spec']['containers'][0]['env']: - if 'GCSFUSE_TOKEN' in env['name']: - continue - elif 'GCSFUSE_TOKENS' in env['name']: - creds.update(env['value']) - else: - env_vars.append(env) - - for f in glob.glob('/home/jovyan/service-account-credentials/*.json'): - bucket = os.path.splitext(os.path.basename(f))[0] - - with open(f, 'r') as f: - creds[bucket] = json.load(f) - - env_vars.append( - {'name': 'GCSFUSE_TOKENS', 'value': json.dumps(creds)}) - - WORKER_TEMPLATE['spec']['containers'][0]['env'] = env_vars - - with open('/home/jovyan/worker-template.yml', 'w') as f: - f.write(yaml.dump(WORKER_TEMPLATE)) - - print('worker-template.yml updated') - - -if __name__ == '__main__': - add_service_creds() \ No newline at end of file diff --git a/notebook/config.yaml b/notebook/config.yaml old mode 100644 new mode 100755 index 3e8ec1a..34dee2a --- a/notebook/config.yaml +++ b/notebook/config.yaml @@ -17,16 +17,10 @@ logging: tornado.application: error # the below section gives defaults for launching clusters from the dask -# labextension viewer. We could customize this but right now it should load -# a default cluster from the worker-template.yml file (as sepecified in the -# following section) +# labextension viewer. We could customize this further labextension: factory: - module: dask_kubernetes - class: KubeCluster + module: dask-gateway + class: GatewayCluster args: [] kwargs: {} - -kubernetes: - worker-template-path: "/home/{NB_USER}/worker-template.yml" - name: "dask-{JUPYTERHUB_USER}-{uuid}" diff --git a/notebook/nbdime_config.json b/notebook/nbdime_config.json new file mode 100644 index 0000000..6bbdaca --- /dev/null +++ b/notebook/nbdime_config.json @@ -0,0 +1,5 @@ +{ + "WebTool": { + "ip": "*" + } +} \ No newline at end of file diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml old mode 100644 new mode 100755 index 81d85f6..9d9d810 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -2,26 +2,36 @@ name: base channels: - conda-forge dependencies: - - black=19.10b0=py37_0 - - coverage=4.5.4=py37h516909a_0 - - dask-labextension=1.0.3=py_0 - - flake8=3.7.9=py37_0 - - ipdb=0.12.3=py_0 - - ipyleaflet=0.11.6=py37_0 - - ipympl=0.3.3=py_0 - - jupyterlab_code_formatter=0.7.0=py_0 - - nano=2.9.8=hb256ff8_1000 - - nb_conda_kernels=2.2.2=py37_0 - - nose=1.3.7=py37_1003 + - black + - bump2version + - coverage + - dask-labextension + - flake8 + - ipdb + - ipyleaflet + - ipympl + - ipypublish + - isort +# need jupyterhub version to match that running on the hub + - jupyterhub=1.2.1 # pinkeep: jupyterhub=1.2.1 + - jupyterlab + - jupyterlab-git + - jupyterlab_code_formatter + - jupyterlab-git + - mypy + - nano + - nbdime + - nb_conda_kernels - octave_kernel=0.31.0=py_0 - oct2py=5.0.4=py_0 - - openssh=7.9p1=h0fa992c_1 - - pip=19.3.1=py37_0 - - pytest=5.3.1=py37_0 - - pytest-cov=2.8.1=py_0 - - pytest-runner=5.2=py_0 - - python-graphviz=0.13.2=py_0 - - sphinx=2.2.2=py_0 - - tox=3.14.2=py_0 + - openssh + - papermill + - pip + - pytest + - pytest-cov + - python-graphviz + - sidecar + - sphinx_rtd_theme + - tox - pip: - black_nbconvert diff --git a/notebook/overrides.json b/notebook/overrides.json old mode 100644 new mode 100755 index 78dcf65..38e8912 --- a/notebook/overrides.json +++ b/notebook/overrides.json @@ -1,8 +1,14 @@ { "@jupyterlab/fileeditor-extension:plugin": { "editorConfig": { - "rulers": [79], + "rulers": [88], "lineNumbers": true } + }, + "@jupyterlab/notebook-extension:tracker": { + "codeCellConfig": { + "rulers": [88], + "lineNumbers": true + } } } diff --git a/notebook/prepare.sh b/notebook/prepare.sh old mode 100644 new mode 100755 index 8d1c835..6b38c04 --- a/notebook/prepare.sh +++ b/notebook/prepare.sh @@ -2,16 +2,14 @@ set -x -echo "Copy Dask configuration files from pre-load directory into home/.config" -mkdir -p /home/jovyan/.config/dask -cp --update -r -v /pre-home/config.yaml /home/jovyan/.config/dask/ +echo "Copy Dask configuration files from pre-load directory into opt/conda/etc/dask/" +mkdir -p /opt/conda/etc/dask +cp --update -r -v /pre-home/config.yaml /opt/conda/etc/dask/ -# should probably pick one of these!!! The second is new, but is implied by the -# cp /pre-home below, and we actually only read the version in ~ in rhg_compute_tools. -cp -r -v /pre-home/worker-template.yml /home/jovyan/.config/dask/ -cp -r -v /pre-home/worker-template.yml /home/jovyan/ +# set credentials for use when starting workers with dask-gateway +python /pre-home/set_gateway_opts.py -sudo rm /pre-home/config.yaml +sudo rm /pre-home/config.yaml /pre-home/set_gateway_opts.py echo "Copy files from pre-load directory into home" cp --update -r -v /pre-home/. /home/jovyan @@ -30,14 +28,15 @@ do echo "Mounting $bucket to /gcs/${bucket}"; mkdir -p "/gcs/$bucket"; /usr/bin/gcsfuse --key-file="$f" "$bucket" "/gcs/${bucket}"; + echo "Including $bucket in dask-gateway options"; fi; fi; done -if [ -f "/home/jovyan/worker-template.yml" ]; then - echo "appending service-account-credentials to worker-template"; - python /home/jovyan/add_service_creds.py; -fi + + +# needed for CLAWPACK to not throw segfaults sometimes +ulimit -s unlimited # Run extra commands $@ diff --git a/notebook/run_sql_proxy.py b/notebook/run_sql_proxy.py deleted file mode 100644 index 7450e01..0000000 --- a/notebook/run_sql_proxy.py +++ /dev/null @@ -1,168 +0,0 @@ -''' -Run an SQL proxy server to enable connections to a cloud sql instance - -To use, create a file at /home/jovyan/setup.cfg with the following contents: - -.. code-block:: bash - - [sql-proxy] - - SQL_INSTANCE = {project}:{region}:{instance}=tcp:{port} - SQL_TOKEN_FILE = /path/to/credentials-file.json - -modifying the `SQL_INSTANCE` and `SQL_TOKEN` values to match your server's -configuration. - -Then, run `python run_sql_proxy.py`. This will start an SQL proxy and will also -add these credentials to your ~/worker-template.yml file. - -When the process is killed (through `^C` or by killing the process) the worker -template will be returned to its previous state. - -''' - -import os -import json -import yaml -import configparser -import subprocess -import signal -import time - - -def get_sql_service_account_token(sql_token_file): - if sql_token_file is None: - return - - try: - with open(sql_token_file, 'r') as f: - return json.load(f) - - except (OSError, IOError): - return - - -class add_sql_proxy_to_worker_spec(object): - kill_now = False - - def __init__(self, sql_instance, sql_token): - self.original_worker_template = None - self.sql_instance = sql_instance - self.sql_token = sql_token - - self.sql_proxy_process = None - - # handle sigint - signal.signal(signal.SIGINT, self.return_worker_spec_to_original_state) - signal.signal(signal.SIGTERM, self.return_worker_spec_to_original_state) - - def __enter__(self): - sql_instance = self.sql_instance - sql_token = self.sql_token - - if (sql_instance is None) or (sql_token is None): - return - - try: - with open('/home/jovyan/worker-template.yml', 'r') as f: - self.original_worker_template = f.read() - worker_template_modified = yaml.safe_load(self.original_worker_template) - - except (OSError, IOError, ValueError): - return - - env_vars = [] - - for env in worker_template_modified['spec']['containers'][0]['env']: - if 'SQL_INSTANCE' in env['name']: - continue - elif 'SQL_TOKEN' in env['name']: - continue - else: - env_vars.append(env) - - env_vars.append( - {'name': 'SQL_TOKEN', 'value': json.dumps(sql_token)}) - - env_vars.append( - {'name': 'SQL_INSTANCE', 'value': sql_instance}) - - worker_template_modified['spec']['containers'][0]['env'] = env_vars - - with open('/home/jovyan/worker-template.yml', 'w') as f: - f.write(yaml.dump(worker_template_modified)) - - print('proxy added to worker-template.yml') - - - def maybe_start_sql_proxy(self, sql_instance, sql_token_file): - if (sql_instance is None) or (sql_token_file is None): - return - - p = subprocess.Popen([ - '/usr/bin/cloud_sql_proxy', - '-instances', - sql_instance, - '-credential_file', - sql_token_file]) - - self.sql_proxy_process = p - - p.wait() - - - def return_worker_spec_to_original_state(self, *args): - if self.original_worker_template is None: - return - - with open('/home/jovyan/worker-template.yml', 'w') as f: - f.write(self.original_worker_template) - - print('proxy removed from worker-template.yml') - - if ( - (self.sql_proxy_process is not None) - and (self.sql_proxy_process.poll() is None)): - - try: - self.sql_proxy_process.kill() - except: - pass - - self.kill_now = True - - - def __exit__(self, *errs): - self.return_worker_spec_to_original_state() - - -def handle_sql_config(): - config = configparser.ConfigParser() - - if not os.path.isfile('/home/jovyan/setup.cfg'): - return - - config.read('/home/jovyan/setup.cfg') - - if not 'sql-proxy' in config.sections(): - return - - sql_instance = config['sql-proxy'].get('SQL_INSTANCE') - sql_token_file = config['sql-proxy'].get('SQL_TOKEN_FILE') - sql_token = get_sql_service_account_token(sql_token_file) - - sql_proxy = add_sql_proxy_to_worker_spec(sql_instance, sql_token) - - with sql_proxy: - sql_proxy.maybe_start_sql_proxy(sql_instance, sql_token_file) - - # wait for sql_proxy to exit gracefully - while True: - if sql_proxy.kill_now: - break - - time.sleep(1) - - -if __name__ == "__main__": - handle_sql_config() diff --git a/notebook/set_gateway_opts.py b/notebook/set_gateway_opts.py new file mode 100644 index 0000000..d48a7d7 --- /dev/null +++ b/notebook/set_gateway_opts.py @@ -0,0 +1,36 @@ +from pathlib import Path +import json +import yaml +import os + +cred_files = Path("/home/jovyan/service-account-credentials").glob("*.json") + +out_file = Path("/opt/conda/etc/dask/gateway.yaml") +out_file.parent.mkdir(exist_ok=True, parents=True) + +# get tokens +tokens = {} +for fpath in cred_files: + bucket = fpath.stem + with open(fpath, "r") as file: + tokens[bucket] = json.load(file) + +# get image names +scheduler_image = os.environ["JUPYTER_IMAGE_SPEC"].replace("/notebook:","/scheduler:") +worker_image = os.environ["JUPYTER_IMAGE_SPEC"].replace("/notebook:","/worker:") + +# create config dict +config = { + "gateway": { + "cluster": { + "options": { + "gcsfuse_tokens": json.dumps(tokens).replace("{","{{").replace("}","}}"), + "scheduler_image": scheduler_image, + "worker_image": worker_image + } + } + } +} + +with open(out_file, "w") as fout: + yaml.safe_dump(config, fout) \ No newline at end of file diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml deleted file mode 100644 index a8990c2..0000000 --- a/notebook/worker-template.yml +++ /dev/null @@ -1,29 +0,0 @@ -metadata: -spec: - restartPolicy: Never - containers: - - args: - - dask-worker - - --nthreads - - '1' - - --no-dashboard - - --memory-limit - - 11.5GB - - --death-timeout - - '60' - env: - - name: GCSFUSE_BUCKET - value: rhg-data - image: rhodium/worker:WILL_BE_REPLACED_BY_TRAVIS - name: dask-worker - securityContext: - capabilities: - add: [SYS_ADMIN] - privileged: true - resources: - limits: - cpu: "1.75" - memory: 11.5G - requests: - cpu: "1.75" - memory: 11.5G diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile index 75039f0..af386d1 100644 --- a/octave-worker/Dockerfile +++ b/octave-worker/Dockerfile @@ -2,11 +2,18 @@ ARG TRAVIS_COMMIT=${TRAVIS_COMMIT} FROM rhodium/worker:${TRAVIS_COMMIT} ARG DEBIAN_FRONTEND=noninteractive +COPY shared_resources /tempdir + ## filepath curation COPY octave_environment.yml /opt/conda/specs/octave_environment.yml ## install octave -RUN sudo apt-get update && sudo apt-get install -yq octave +RUN apt-get update \ + && apt-get install -yq --no-install-recommends octave octave-optim # add octave-specific packages RUN conda env update -f /opt/conda/specs/octave_environment.yml + +RUN rm -rf /tempdir \ + && conda clean -yaf \ + && sudo apt-get clean diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml old mode 100644 new mode 100755 diff --git a/pin.py b/pin.py index 175ab22..37ce9eb 100644 --- a/pin.py +++ b/pin.py @@ -25,6 +25,15 @@ from ruamel.yaml import YAML +SPEC_FILES = [ + ('shared_resources/base_environment.yml', 'base'), + ('shared_resources/scheduler_environment.yml', 'base'), + ('notebook/notebook_environment.yml', 'base'), + ('octave-worker/octave_environment.yml', 'base'), + ('notebook/r_environment.yml', 'r'), +] + + def get_versions_in_current_environment(envname='base'): ''' Calls ``conda env export -n {envname} --json`` and returns spec @@ -294,26 +303,18 @@ def pinversions(): def pin(file, dry_run): '''Pin packages in environment files based on environments on the local machine''' - spec_files = [ - ('base_environment.yml', 'base'), - ('notebook/notebook_environment.yml', 'base'), - ('octave-worker/octave_environment.yml', 'base'), - ('notebook/r_environment.yml', 'r')] - if file == 'all': - pin_files(spec_files, dry_run=dry_run) + pin_files(SPEC_FILES, dry_run=dry_run) elif file == 'base': - pin_files([spec_files[0]], dry_run=dry_run) + pin_files([SPEC_FILES[0]], dry_run=dry_run) elif file == 'notebook': - pin_files([spec_files[1]], dry_run=dry_run) + pin_files([SPEC_FILES[1]], dry_run=dry_run) elif file == 'octave': - pin_files([spec_files[2]], dry_run=dry_run) - elif file == 'r': - pin_files([spec_files[3]], dry_run=dry_run) + pin_files([SPEC_FILES[2]], dry_run=dry_run) else: raise ValueError( 'env type not recognized: {}' - 'choose from "base", "notebook", "octave", "r", or "all".' + 'choose from "base", "notebook", "octave", or "all".' .format(file)) @@ -329,26 +330,22 @@ def pin(file, dry_run): def unpin(file, dry_run): '''Unpin packages in environment files''' - spec_files = [ - ('base_environment.yml', 'base'), - ('notebook/notebook_environment.yml', 'base'), - ('octave-worker/octave_environment.yml', 'base'), - ('notebook/r_environment.yml', 'r')] - if file == 'all': - unpin_files(spec_files, dry_run=dry_run) + unpin_files(SPEC_FILES, dry_run=dry_run) elif file == 'base': - unpin_files([spec_files[0]], dry_run=dry_run) + unpin_files([SPEC_FILES[0]], dry_run=dry_run) + elif file == 'scheduler': + unpin_files([SPEC_FILES[1]], dry_run=dry_run) elif file == 'notebook': - unpin_files([spec_files[1]], dry_run=dry_run) + unpin_files([SPEC_FILES[2]], dry_run=dry_run) elif file == 'octave': - unpin_files([spec_files[2]], dry_run=dry_run) + unpin_files([SPEC_FILES[3]], dry_run=dry_run) elif file == 'r': - unpin_files([spec_files[3]], dry_run=dry_run) + unpin_files([SPEC_FILES[4]], dry_run=dry_run) else: raise ValueError( 'env type not recognized: {}' - 'choose from "base", "notebook", "octave", "r", or "all".' + 'choose from "base", "scheduler", "notebook", "octave", "r", or "all".' .format(file)) diff --git a/base_environment.yml b/shared_resources/base_environment.yml old mode 100644 new mode 100755 similarity index 73% rename from base_environment.yml rename to shared_resources/base_environment.yml index ea54de9..990c316 --- a/base_environment.yml +++ b/shared_resources/base_environment.yml @@ -11,13 +11,11 @@ dependencies: - cartopy=0.17.0=py37h423102d_1009 - cftime=1.0.4.2=py37hc1659b7_0 - click=7.0=py_0 - - compilers=1.0.4=0 - - dask=2.8.1=py_0 + - compilers=1.1.1=0 - dask-glm=0.2.0=py_1 - dask-ml=1.1.1=py_0 - datashader=0.8.0=py_0 - descartes=1.1.0=py_4 - - distributed=2.8.1=py_0 - dropbox=9.4.0=py_0 # need to make sure we get esmpy compiled with mpi otherwise xesmf regridding # won't work @@ -25,11 +23,6 @@ dependencies: - fastparquet=0.3.2=py37hc1659b7_0 - fiona=1.8.13=py37h900e953_0 - fusepy=3.0.1=py_0 -# this gcc pin is necessary b/c of a weird feature in the h553295d_15 build -# which makes it hard to build numpy-based cython extensions (like pyclaw). -# we should try removing it whenever we next do an update and see if Clawpack -# can still be built - - gcc_linux-64=7.3.0=h553295d_14 # pinkeep: gcc_linux-64=7.3.0=h553295d_14 - gcsfs=0.5.3=py_0 - gdal=3.0.2=py37hbb6b9fb_5 - geoalchemy2=0.6.3=py_0 @@ -38,19 +31,17 @@ dependencies: - geotiff=1.5.1=hbd99317_7 - geoviews=1.6.6=py_0 - git=2.24.0=pl526hce37bd2_1 - - gitpython=3.0.5=py_0 + - gitpython=3.1.8=py_0 - google-cloud-container=0.3.0=py37_0 - google-cloud-storage=1.23.0=py37_0 - holoviews=1.12.7=py_0 - h5netcdf=0.7.4=py_0 - icu=64.2=he1b5a44_1 + - intake-esm=2020.6.11 - iris=2.2.0=py37_1003 - jedi=0.15.1=py37_0 -# need server proxy on workers if using remote scheduler - - jupyter-server-proxy=1.3.2=py_0 - kubernetes - lapack=3.6.1=ha44fe06_2 - - lz4=2.2.1=py37hd79334b_0 - make=4.2.1=h14c3975_2004 - matplotlib=3.1.2=py37_1 - nc-time-axis=1.2.0=py_0 @@ -59,12 +50,12 @@ dependencies: - netcdf4=1.5.3=mpi_mpich_py37h01ee55b_1 - numba=0.46.0=py37hb3f55d8_1 - numcodecs=0.6.4=py37he1b5a44_0 - - pandas=0.25.3=py37hb3f55d8_0 # for geoviews - phantomjs=2.1.1=1 - pip=19.3.1=py37_0 - plotly=4.3.0=py_0 - polyline=1.4.0=py_0 + - pydap=3.2.2=py37_1000 - pygeos=0.5=py37h5d51c17_1 - pyinterp=0.0.7=py37h97f2665_0 - pyshp=2.1.0=py_0 @@ -85,10 +76,9 @@ dependencies: - shapely=1.6.4=py37h5d51c17_1007 - sparse=0.8.0=py_0 - statsmodels=0.10.2=py37hc1659b7_0 - - tini=0.18.0=h14c3975_1001 - unzip=6.0=h516909a_0 - uritemplate=3.0.0=py_1 - - xarray=0.14.1=py_0 + - xarray=0.16.0=py_0 - xesmf=0.2.1=py_0 - xgcm=0.2.0=py_0 - xhistogram=0.1.1=py_0 @@ -104,9 +94,4 @@ dependencies: - climate-toolbox==0.1.5 - impactlab-tools==0.4.0 - parameterize-jobs==0.1.1 - - rhg_compute_tools==0.2.2 - - git+https://github.com/NCAR/intake-esm.git@v2020.3.16.2#egg=intake_esm -# need to install from master until 0.10.1 -# due to handling of remote scheduler -# (we also should at some point switch to dask-gateway instead of dask-kubernetes) - - dask_kubernetes==0.10.1 + - git+https://github.com/RhodiumGroup/rhg_compute_tools.git@gateway#egg=rhg_compute_tools # needed for dask-gateway until merged into master \ No newline at end of file diff --git a/common.sh b/shared_resources/common.sh old mode 100644 new mode 100755 similarity index 84% rename from common.sh rename to shared_resources/common.sh index 3078d6e..21e2eb8 --- a/common.sh +++ b/shared_resources/common.sh @@ -5,8 +5,10 @@ apt-get install -yq --no-install-recommends \ apt-utils \ zip \ bzip2 \ + unzip \ ca-certificates \ curl \ + locales \ lsb-release \ gnupg2 \ sudo \ @@ -38,8 +40,3 @@ chmod +x /usr/bin/cloud_sql_proxy chmod +x /usr/bin/prepare.sh mkdir /gcs mkdir /opt/app - -# super sketchy hack to get around our need for compiler_compat binaries and some -# other things that cause problems together? -# see https://github.com/ContinuumIO/anaconda-issues/issues/11152 -rm -rf /opt/conda/compiler_compat/ld diff --git a/shared_resources/fix-permissions.sh b/shared_resources/fix-permissions.sh new file mode 100755 index 0000000..c8d0032 --- /dev/null +++ b/shared_resources/fix-permissions.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +## Adapted from: +## https://github.com/jupyter/docker-stacks/blob/master/base-notebook/fix-permissions + +# set permissions on a directory +# after any installation, if a directory needs to be (human) user-writable, +# run this script on it. +# It will make everything in the directory owned by the group $NB_GID +# and writable by that group. +# Deployments that want to set a specific user id can preserve permissions +# by adding the `--group-add users` line to `docker run`. + +# uses find to avoid touching files that already have the right permissions, +# which would cause massive image explosion + +# right permissions are: +# group=$NB_GID +# AND permissions include group rwX (directory-execute) +# AND directories have setuid,setgid bits set + +set -e + +for d in "$@"; do + find "$d" \ + ! \( \ + -group $NB_GID \ + -a -perm -g+rwX \ + \) \ + -exec chgrp $NB_GID {} \; \ + -exec chmod g+rwX {} \; + # setuid,setgid *on directories only* + find "$d" \ + \( \ + -type d \ + -a ! -perm -6000 \ + \) \ + -exec chmod +6000 {} \; +done \ No newline at end of file diff --git a/shared_resources/scheduler_environment.yml b/shared_resources/scheduler_environment.yml new file mode 100644 index 0000000..f08e3bc --- /dev/null +++ b/shared_resources/scheduler_environment.yml @@ -0,0 +1,15 @@ +# Only the packages necessary to build a dask-gateway scheduler. As seen here: +# https://github.com/dask/dask-gateway/blob/master/dask-gateway/Dockerfile +channels: + - conda-forge +dependencies: + - aiohttp=3.7.3=py38h25fe258_0 + - dask=2.30.0=py_0 + - distributed=2.30.1=py38h578d9bd_0 + - dask-gateway=0.9.0=py38h578d9bd_0 + - lz4=3.1.1=py38h87b837d_0 + - jupyter-server-proxy=1.5.0=py_0 + - numpy=1.19.4=py38hf0fd68c_1 + - pandas=1.1.4=py38h0ef3d22_0 + - tini=0.18.0=h14c3975_1001 +name: base \ No newline at end of file diff --git a/worker/Dockerfile b/worker/Dockerfile index 13ae443..8981dba 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -1,56 +1,14 @@ -## using same container as jupyter/base-notebook:python-3.7.3 -FROM ubuntu:bionic-20190612@sha256:6e9f67fa63b0323e9a1e587fd71c561ba48a034504fb804fd26fd8800039835d +## using same base image as the "base image for the base image" of the notebook image +FROM rhodium/scheduler:local ARG DEBIAN_FRONTEND=noninteractive -ENV NPY_DISTUTILS_APPEND_FLAGS=1 - -## filepath curation -RUN mkdir /tempdir -COPY common.sh /tempdir - -COPY add_service_creds.py /usr/bin -COPY prepare.sh /usr/bin - ## perform a bunch of common actions RUN bash /tempdir/common.sh - -########### -## install miniconda -# adapted from continuumio/miniconda3 - -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV PATH /opt/conda/bin:$PATH - -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc -########### - - -## set up conda channels -RUN mkdir /opt/conda/specs -COPY base_environment.yml /opt/conda/specs -RUN conda config --add channels conda-forge/label/dev && \ - conda config --add channels conda-forge - - -## set up python env -RUN conda update -n base conda RUN conda env update -f /opt/conda/specs/base_environment.yml RUN conda list -n base ## clean up -RUN rm -rf /var/lib/apt/lists/* /tempdir -RUN conda clean --all -f - - -## prepare container -ENV OMP_NUM_THREADS=1 -ENV MKL_NUM_THREADS=1 -ENV OPENBLAS_NUM_THREADS=1 - -ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] +RUN rm -rf /var/lib/apt/lists/* /tempdir \ + && conda clean -yaf \ + && sudo apt-get clean \ No newline at end of file diff --git a/worker/Dockerfile_scheduler b/worker/Dockerfile_scheduler new file mode 100644 index 0000000..2a0030c --- /dev/null +++ b/worker/Dockerfile_scheduler @@ -0,0 +1,72 @@ +## using same base image as the "base image for the base image" of the notebook image +FROM ubuntu:bionic-20190612@sha256:6e9f67fa63b0323e9a1e587fd71c561ba48a034504fb804fd26fd8800039835d +ARG DEBIAN_FRONTEND=noninteractive + +# needed to properly install packages that use numpy libraries +ENV NPY_DISTUTILS_APPEND_FLAGS=1 + +# needed so that matplotlib will work headless +ENV MPLBACKEND=Agg + +## filepath curation +COPY shared_resources /tempdir +COPY add_service_creds.py /usr/bin +COPY prepare.sh /usr/bin + +# install apt-get packages +RUN apt-get update -y --no-install-recommends +RUN apt-get install -yq --no-install-recommends \ + ca-certificates \ + locales \ + sudo \ + wget + + +########### +## install miniconda (following jupyter/base-notebook +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV CONDA_DIR=/opt/conda \ + SHELL=/bin/bash \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 +ENV PATH=$CONDA_DIR/bin:$PATH + +ENV MINICONDA_VERSION=4.7.12.1 \ + MINICONDA_MD5=81c773ff87af5cfac79ab862942ab6b3 + +WORKDIR /tmp +RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ + echo "${MINICONDA_MD5} *Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \ + /bin/bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ + rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ + conda config --system --prepend channels conda-forge && \ + conda config --system --set channel_priority strict && \ + conda update -n base --yes conda && \ + conda update --all --yes +########### + +## set up python env +RUN mkdir /opt/conda/specs +COPY shared_resources/scheduler_environment.yml /opt/conda/specs +COPY shared_resources/base_environment.yml /opt/conda/specs +RUN conda env update -f /opt/conda/specs/scheduler_environment.yml + +## clean up +RUN rm -rf /var/lib/apt/lists/* /tmp/* \ + && conda clean -yaf \ + && sudo apt-get clean + +## prepare container +ENV OMP_NUM_THREADS=1 +ENV MKL_NUM_THREADS=1 +ENV OPENBLAS_NUM_THREADS=1 + +WORKDIR / + +RUN chmod +x /usr/bin/prepare.sh + +ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] \ No newline at end of file