From 1d5063f681b7aee0161d3f542b613dd20135f7c8 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Mon, 23 Sep 2019 22:54:28 -0700 Subject: [PATCH 001/156] add package alignment tests --- .gitignore | 7 +- pytest.ini | 3 + tests/test_package_alignment.py | 179 ++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 pytest.ini create mode 100644 tests/test_package_alignment.py diff --git a/.gitignore b/.gitignore index f7bad50..79e93af 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,9 @@ jupyter-config.yml secret-config.yml storageclass.yml values.yml -worker-config.yml \ No newline at end of file +worker-config.yml + +# tests +.coverage +__pycache__ +.pytest_cache diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..d5c053e --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths=./tests +addopts= --cov=tests --doctest-modules --cov-report term-missing diff --git a/tests/test_package_alignment.py b/tests/test_package_alignment.py new file mode 100644 index 0000000..b324880 --- /dev/null +++ b/tests/test_package_alignment.py @@ -0,0 +1,179 @@ + +import os +import re +import pprint +import conda.core.solve +import pytest + +PKG_ROOT = os.path.dirname(os.path.dirname(__file__)) + +IMAGES_TO_CHECK = [ + 'notebook', + 'worker', + 'octave-worker', + ] + +PAIRINGS = { + 'notebook': { + 'base': ['worker'], + 'octave': ['octave-worker'], + }, +} + +CONDA_ARGS = { + 'n': 'name', + 'p': 'prefix', + 'c': 'channel', + 'S': 'satisfied-skip-solve', + 'm': 'mkdir', + 'C': 'use-index-cache', + 'k': 'insecure', + 'd': 'dry-run', + 'q': 'quiet', + 'v': 'verbose', + 'y': 'yes'} + +def get_conda_specs(dockerfile): + with open(dockerfile, 'r') as f: + conda_specs = [] + + line_getter = f + + in_conda = False + env = 'base' + + while True: + try: + line = next(f).split('#')[0].strip() + except StopIteration: + break + + args = re.split(r'\s+', line.rstrip('\\').strip()) + + if ( + (args[0].upper() == 'RUN') + and (args[1].lower() in ['conda', 'source']) + and (args[2].lower() == 'activate')): + env = args[3] + + elif ( + (args[0].upper() == 'RUN') + and (args[1].lower() == 'conda') + and (args[2].lower() in ['install', 'create'])): + + conda_spec = args[1:] + + if line.endswith('\\'): + in_conda = True + else: + conda_specs.append((env, conda_spec)) + + elif in_conda: + conda_spec += args + + if not line.endswith('\\'): + conda_specs.append((env, conda_spec)) + in_conda = False + + return conda_specs + +def parse_conda_create_command(env, args): + if args[0].strip().lower() != 'conda': + return ('invalid', args) + + chaffe = ['conda', 'install', 'update', 'create', '-y', '--yes'] + install_args = [a for a in args if a.lower() not in chaffe] + if len(install_args) == 1 and install_args[0].split('=')[0] == 'conda': + return {'upgrade': tuple(install_args[0].strip('"\'').split('='))} + + spec = {'packages': {}} + + get_install_args = iter(install_args) + + while True: + try: + arg = next(get_install_args) + except StopIteration: + break + + if arg.startswith('-'): + a = arg.strip('-') + spec[CONDA_ARGS.get(a, a)] = next(get_install_args) + + else: + pkgver = arg.strip('"\'').split('=') + if len(pkgver) == 1: + pkgver.append(None) + spec['packages'].update({pkgver[0]: pkgver[1]}) + + spec['name'] = spec.get('name', env) + spec['channel'] = spec.get('channel', 'defaults') + + return spec + +def assert_pairing_match(env, base, penv, pairs): + for base_spec in base: + for pkg, ver in base_spec['packages'].items(): + for pair in pairs: + if pkg in pair.get('packages', {}): + pair_ver = pair['packages'][pkg] + test = (ver == pair_ver) + msg = ( + 'Package versions mis-aligned in env {}:{}' + '\n\tpackage {}: {} != {}' + .format(env, penv, pkg, ver, pair_ver)) + assert test, msg + +def assert_conda_spec_validity(img, spec): + for img_spec in spec: + if not 'packages' in img_spec: + continue + + # for s, cmd in img_spec.items(): + print( + 'Solving {} package spec {} with channel {}'.format( + img, + img_spec.get('name', 'base'), + img_spec.get('channel', 'defaults'))) + + solver = conda.core.solve.Solver( + img_spec.get('name', 'base'), + channels=[img_spec.get('channel', 'defaults'), 'defaults'], + specs_to_add=tuple(map(lambda x: '='.join([i for i in x if i is not None]), img_spec['packages'].items()))) + + state = solver.solve_final_state() + + +@pytest.fixture(scope='module') +def package_spec(): + specs = {} + + for img_name in IMAGES_TO_CHECK: + files = os.listdir(os.path.join(PKG_ROOT, img_name)) + dockerfiles = [f for f in files if f.startswith('Dockerfile')] + specs[img_name] = [] + for dockerfile in dockerfiles: + fp = os.path.join(PKG_ROOT, img_name, dockerfile) + s = get_conda_specs(fp) + env_specs = [ + parse_conda_create_command(k, v) + for (k, v) in get_conda_specs(fp)] + + specs[img_name] += env_specs + + yield specs + + +@pytest.mark.parametrize('base', PAIRINGS.keys()) +def test_package_pairing(package_spec, base): + pairs = PAIRINGS[base] + for env, paired in pairs.items(): + for p in paired: + nb_env = [s for s in package_spec[base] if 'name' in s and s['name'] == env] + assert_pairing_match(env, nb_env, p, package_spec[p]) + + +@pytest.mark.parametrize('img', IMAGES_TO_CHECK) +def test_package_validity(package_spec, img): + spec = package_spec[img] + assert_conda_spec_validity(img, spec) From b0d4493a23f1914bff0870e0194765dcae43e3a6 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 17:59:05 -0700 Subject: [PATCH 002/156] test alignment --- tests/test_package_alignment.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_package_alignment.py b/tests/test_package_alignment.py index b324880..4e75457 100644 --- a/tests/test_package_alignment.py +++ b/tests/test_package_alignment.py @@ -10,13 +10,11 @@ IMAGES_TO_CHECK = [ 'notebook', 'worker', - 'octave-worker', ] PAIRINGS = { 'notebook': { 'base': ['worker'], - 'octave': ['octave-worker'], }, } From ccaced20fb1ba87682c1696a87d0a13d685bed4f Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 17:59:43 -0700 Subject: [PATCH 003/156] move conda specs to environment files --- notebook/Dockerfile | 116 +------------------------------------ notebook/environment.yml | 93 +++++++++++++++++++++++++++++ notebook/environment_r.yml | 25 ++++++++ worker/Dockerfile | 1 - worker/environment.yml | 88 ++++++++++++++++++++++++++++ 5 files changed, 208 insertions(+), 115 deletions(-) create mode 100644 notebook/environment.yml create mode 100644 notebook/environment_r.yml create mode 100644 worker/environment.yml diff --git a/notebook/Dockerfile b/notebook/Dockerfile index d8e87a6..7367b7d 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -6,120 +6,8 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -RUN conda install --yes -c conda-forge \ - basemap=1.1.0 \ - beautifulsoup4=4.6.1 \ - bokeh=0.13.0 \ - bqplot \ - cartopy=0.16.0 \ - click=6.7 \ - cloudpickle=0.5.3 \ - cytoolz=0.9.0.1 \ - dask-core=0.19.4 \ - dask-glm=0.1.0 \ - dask-ml=0.11.0 \ - dask-searchcv=0.2.0 \ - dask=0.19.4 \ - datashader=0.6.8 \ - distributed=1.23.3 \ - esmpy=7.1.0r \ - fastparquet=0.1.6 \ - fiona=1.7.13 \ - fusepy=2.0.4 \ - gcsfs=0.1.2 \ - gdal=2.2.4 \ - geoalchemy2=0.5.0 \ - geopy=1.17.0 \ - geoviews=1.5.1 \ - git=2.18.0 \ - gitpython=2.1.11 \ - holoviews=1.10.7 \ - ipdb=0.11 \ - ipywidgets=7.4.2 \ - jedi=0.12.0 \ - jupyter=1.0.0 \ - jupyterhub=0.9.4 \ - jupyterlab=0.35.2 \ - lz4-c=1.8.1.2 \ - lz4=1.1.0 \ - matplotlib=2.2.2 \ - nb_conda_kernels=2.1.1 \ - nbserverproxy=0.8.3 \ - ncurses=6.1 \ - netcdf-fortran=4.4.4 \ - netcdf4=1.3.1 \ - nomkl=2.0 \ - nose=1.3.7 \ - numba=0.37.0 \ - numcodecs=0.5.5 \ - numpy=1.14.2 \ - pandas=0.23.4 \ - pip=9.0.3 \ - pyshp=1.2.12 \ - python-blosc=1.4.4 \ - python-snappy=0.5.3 \ - pyviz_comms=0.6.0 \ - PyYAML=3.12 \ - rasterio=0.36.0 \ - scikit-image=0.14.1 \ - scikit-learn=0.20.0 \ - scipy=1.1.0 \ - seaborn=0.9.0 \ - setuptools=39.2.0 \ - statsmodels=0.9.0 \ - tornado=5.0.2 \ - tqdm=4.24.0 \ - urllib3=1.23 \ - xarray=0.10.9 \ - zarr=2.2.0 \ - zict=0.1.3 - -RUN conda install --yes --channel conda-forge/label/dev geopandas - -#install R packages -RUN conda install --quiet --yes \ - 'rpy2=2.8*' \ - 'r-base=3.4.1' \ - 'r-irkernel=0.8*' \ - 'r-plyr=1.8*' \ - 'r-devtools=1.13*' \ - 'r-tidyverse=1.1*' \ - 'r-shiny=1.0*' \ - 'r-rmarkdown=1.8*' \ - 'r-forecast=8.2*' \ - 'r-rsqlite=2.0*' \ - 'r-reshape2=1.4*' \ - 'r-nycflights13=0.2*' \ - 'r-caret=6.0*' \ - 'r-rcurl=1.95*' \ - 'r-crayon=1.3*' \ - 'r-randomforest=4.6*' \ - 'r-htmltools=0.3*' \ - 'r-sparklyr=0.7*' \ - 'r-htmlwidgets=1.0*' \ - 'r-hexbin=1.27*' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip install \ - bumpversion==0.5.3 \ - coverage==4.5.1 \ - git+git://github.com/dask/dask-kubernetes.git@295f06428119bc8e674a2e6a59cd97000fae69e8 \ - flake8==3.5.0 \ - google-cloud-storage==1.12.0 \ - kubernetes==6.0.0 \ - mapbox==0.16.2 \ - py-noaa==0.2.2 \ - pytest-cov==2.5.1 \ - pytest-runner==4.2 \ - pytest==3.6.2 \ - Sphinx==1.7.5 \ - tox==3.1.1 \ - wget==3.2 \ - wheel==0.31.1 \ - xesmf==0.1.1 \ - xlrd==1.1.0 \ - --no-cache-dir +RUN conda env update -n base --yes -f environment.yml +RUN conda env update -n r --yes -f environment_r.yml RUN jupyter labextension install \ @jupyter-widgets/jupyterlab-manager@0.38 \ diff --git a/notebook/environment.yml b/notebook/environment.yml new file mode 100644 index 0000000..dae9a39 --- /dev/null +++ b/notebook/environment.yml @@ -0,0 +1,93 @@ +name: notebook +channels: + - conda-forge +dependencies: + - beautifulsoup4=4.8.0 + - bokeh=1.3.4 + - bqplot=0.11.8 + - cartopy=0.17.0 + - click + - cloudpickle=1.2.2 + - cytoolz=0.10.0 + - dask-core=2.4.0 + - dask-glm=0.2.0 + - dask-ml=1.0.0 + - dask=2.4.0 + - datashader=0.7.0 + - distributed=2.4.0 + - esmf=7.1.0 + - esmpy=7.1.0 + - fastparquet=0.3.2 + - fiona=1.8.6 + - fusepy + - gcsfs=0.3.0 + - gdal=2.4.2 + - geoalchemy2=0.6.3 + - geopandas=0.5.1 + - geopy=1.20.0 + - geotiff=1.5.1 + - geoviews=1.6.3 + - git=2.23.0 + - gitpython=3.0.2 + - holoviews=1.12.5 + - ipdb=0.12.2 + - ipython=7.8.0 + - ipywidgets=7.5.1 + - jedi=0.15.1 + - jupyter_client=5.3.3 + - jupyter=1.0.0 + - jupyterhub=1.0.0 + - jupyterlab=1.1.4 + - lz4=2.2.1 + - matplotlib=3.1.1 + - nb_conda_kernels=2.2 + - nbconvert=5.6.0 + - nbserverproxy + - ncurses + - netcdf-fortran + - netcdf4=1.5.1.2 + - nose + - numba=0.45.1 + - numcodecs=0.6.3 + - numpy + - pandas + - pip + - pyshp=2.1.0 + - python-blosc=1.8.1 + - python-snappy=0.5.4 + - pyviz_comms=0.7.2 + - PyYAML + - rasterio=1.0.28 + - scikit-image=0.15.0 + - scikit-learn=0.21.3 + - scipy=1.3.1 + - seaborn=0.9.0 + - shapely=1.6.4 + - statsmodels=0.10.1 + - toolz=0.10.0 + - tornado=6.0.3 + - tqdm=4.36.1 + - urllib3=1.25.6 + - widgetsnbextension=3 + - xarray=0.13.0 + - zarr=2.3.2 + - zeromq=4.3.2 + - zict=1.0.0 + - pip: + - bumpversion==0.5.3 + - coverage==4.5.1 + - git+git://github.com/dask/dask-kubernetes.git@295f06428119bc8e674a2e6a59cd97000fae69e8 + - flake8==3.5.0 + - google-cloud-storage==1.12.0 + - kubernetes==6.0.0 + - mapbox==0.16.2 + - py-noaa==0.2.2 + - pytest-cov==2.5.1 + - pytest-runner==4.2 + - pytest==3.6.2 + - Sphinx==1.7.5 + - tox==3.1.1 + - wget==3.2 + - wheel==0.31.1 + - xesmf==0.1.1 + - xlrd==1.1.0 diff --git a/notebook/environment_r.yml b/notebook/environment_r.yml new file mode 100644 index 0000000..8989b16 --- /dev/null +++ b/notebook/environment_r.yml @@ -0,0 +1,25 @@ +name: r +channels: + - conda-forge + - r +dependencies: + - rpy2=2.8* + - r-base=3.4.1 + - r-irkernel=0.8* + - r-plyr=1.8* + - r-devtools=1.13* + - r-tidyverse=1.1* + - r-shiny=1.0* + - r-rmarkdown=1.8* + - r-forecast=8.2* + - r-rsqlite=2.0* + - r-reshape2=1.4* + - r-nycflights13=0.2* + - r-caret=6.0* + - r-rcurl=1.95* + - r-crayon=1.3* + - r-randomforest=4.6* + - r-htmltools=0.3* + - r-sparklyr=0.7* + - r-htmlwidgets=1.0* + - r-hexbin=1.27* diff --git a/worker/Dockerfile b/worker/Dockerfile index 8b345f8..4bc8f9d 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -7,7 +7,6 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda RUN conda create -n worker --yes -c conda-forge \ - basemap=1.1.0 \ beautifulsoup4=4.6.1 \ bokeh=0.13.0 \ bqplot \ diff --git a/worker/environment.yml b/worker/environment.yml new file mode 100644 index 0000000..f024376 --- /dev/null +++ b/worker/environment.yml @@ -0,0 +1,88 @@ +name: worker +channels: + - conda-forge +dependencies: + - beautifulsoup4=4.8.0 + - bokeh=1.3.4 + - bqplot=0.11.8 + - cartopy=0.17.0 + - click + - cloudpickle=1.2.2 + - cytoolz=0.10.0 + - dask-core=2.4.0 + - dask-glm=0.2.0 + - dask-ml=1.0.0 + - dask=2.4.0 + - datashader=0.7.0 + - distributed=2.4.0 + - esmf=7.1.0 + - esmpy=7.1.0 + - fastparquet=0.3.2 + - fiona=1.8.6 + - fusepy + - gcsfs=0.3.0 + - gdal=2.4.2 + - geoalchemy2=0.6.3 + - geopandas=0.5.1 + - geopy=1.20.0 + - geotiff=1.5.1 + - geoviews=1.6.3 + - git=2.23.0 + - gitpython=3.0.2 + - holoviews=1.12.5 + - ipdb=0.12.2 + - ipython=7.8.0 + - ipywidgets=7.5.1 + - jedi=0.15.1 + - jupyter_client=5.3.3 + - jupyter=1.0.0 + - lz4=2.2.1 + - matplotlib=3.1.1 + - nb_conda_kernels=2.2 + - ncurses + - netcdf-fortran + - netcdf4=1.5.1.2 + - nose + - numba=0.45.1 + - numcodecs=0.6.3 + - numpy + - pandas + - pip + - pyshp=2.1.0 + - python-blosc=1.8.1 + - python-snappy=0.5.4 + - pyviz_comms=0.7.2 + - PyYAML + - rasterio=1.0.28 + - scikit-image=0.15.0 + - scikit-learn=0.21.3 + - scipy=1.3.1 + - seaborn=0.9.0 + - shapely=1.6.4 + - statsmodels=0.10.1 + - toolz=0.10.0 + - tornado=6.0.3 + - tqdm=4.36.1 + - urllib3=1.25.6 + - xarray=0.13.0 + - zarr=2.3.2 + - zeromq=4.3.2 + - zict=1.0.0 + - pip: + - bumpversion==0.5.3 + - coverage==4.5.1 + - git+git://github.com/dask/dask-kubernetes.git@295f06428119bc8e674a2e6a59cd97000fae69e8 + - flake8==3.5.0 + - google-cloud-storage==1.12.0 + - kubernetes==6.0.0 + - mapbox==0.16.2 + - py-noaa==0.2.2 + - pytest-cov==2.5.1 + - pytest-runner==4.2 + - pytest==3.6.2 + - Sphinx==1.7.5 + - tox==3.1.1 + - wget==3.2 + - wheel==0.31.1 + - xesmf==0.1.1 + - xlrd==1.1.0 From 2189974d2dc6c9f822742b52b1a2b989a5f7c734 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 18:25:14 -0700 Subject: [PATCH 004/156] worker build to env file --- worker/Dockerfile | 77 +---------------------------------------------- 1 file changed, 1 insertion(+), 76 deletions(-) diff --git a/worker/Dockerfile b/worker/Dockerfile index 4bc8f9d..68d4c2e 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -6,73 +6,7 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -RUN conda create -n worker --yes -c conda-forge \ - beautifulsoup4=4.6.1 \ - bokeh=0.13.0 \ - bqplot \ - cartopy=0.16.0 \ - click=6.7 \ - cloudpickle=0.5.3 \ - cytoolz=0.9.0.1 \ - dask-core=0.19.4 \ - dask-glm=0.1.0 \ - dask-ml=0.11.0 \ - dask-searchcv=0.2.0 \ - dask=0.19.4 \ - datashader=0.6.8 \ - distributed=1.23.3 \ - esmpy=7.1.0r \ - fastparquet=0.1.6 \ - fiona=1.7.13 \ - fusepy=2.0.4 \ - gcsfs=0.1.2 \ - gdal=2.2.4 \ - geoalchemy2=0.5.0 \ - geopy=1.17.0 \ - geoviews=1.5.1 \ - git=2.18.0 \ - gitpython=2.1.11 \ - holoviews=1.10.7 \ - ipdb=0.11 \ - ipywidgets=7.4.2 \ - jedi=0.12.0 \ - jupyter=1.0.0 \ - lz4-c=1.8.1.2 \ - lz4=1.1.0 \ - matplotlib=2.2.2 \ - nb_conda_kernels=2.1.1 \ - nbserverproxy=0.8.3 \ - ncurses=6.1 \ - netcdf-fortran=4.4.4 \ - netcdf4=1.3.1 \ - nomkl=2.0 \ - nose=1.3.7 \ - numba=0.37.0 \ - numcodecs=0.5.5 \ - numpy=1.14.2 \ - pandas=0.23.4 \ - pip=9.0.3 \ - pyshp=1.2.12 \ - python-blosc=1.4.4 \ - python-snappy=0.5.3 \ - pyviz_comms=0.6.0 \ - python=3.6 \ - PyYAML=3.12 \ - rasterio=0.36.0 \ - scikit-image=0.14.1 \ - scikit-learn=0.20.0 \ - scipy=1.1.0 \ - seaborn=0.9.0 \ - setuptools=39.2.0 \ - statsmodels=0.9.0 \ - tornado=5.0.2 \ - tqdm=4.24.0 \ - urllib3=1.23 \ - xarray=0.10.9 \ - zarr=2.2.0 \ - zict=0.1.3 - -RUN conda install -n worker --yes --channel conda-forge/label/dev geopandas +RUN conda env update -n worker --yes -f environment.yml RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ @@ -86,15 +20,6 @@ RUN export GCSFUSE_REPO=gcsfuse-xenial \ && apt-get install gcsfuse \ && alias googlefuse=/usr/bin/gcsfuse -RUN /opt/conda/envs/worker/bin/pip install \ - git+git://github.com/dask/dask-kubernetes.git@295f06428119bc8e674a2e6a59cd97000fae69e8 \ - google-cloud-storage==1.12.0 \ - pyasn1-modules==0.2.2 \ - pyasn1==0.4.3 \ - wget==3.2 \ - xesmf==0.1.1 \ - --no-cache-dir - # clean up RUN apt-get clean && rm -rf /var/lib/apt/lists/* RUN conda clean -tipsy From 4f499a5549073ef0055d4fdb2944929b44e18d21 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 18:43:57 -0700 Subject: [PATCH 005/156] bug fix in conda install --- notebook/Dockerfile | 4 ++-- worker/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 7367b7d..52eb06b 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -6,8 +6,8 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -RUN conda env update -n base --yes -f environment.yml -RUN conda env update -n r --yes -f environment_r.yml +RUN conda env update -n base -f environment.yml +RUN conda env create -n r -f environment_r.yml RUN jupyter labextension install \ @jupyter-widgets/jupyterlab-manager@0.38 \ diff --git a/worker/Dockerfile b/worker/Dockerfile index 68d4c2e..3974844 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -6,7 +6,7 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -RUN conda env update -n worker --yes -f environment.yml +RUN conda env create -n worker -f environment.yml RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ From 59407c8ce33fd6180b44f9add857e1f7b395ae1e Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 19:13:45 -0700 Subject: [PATCH 006/156] another bug fix in conda build --- notebook/Dockerfile | 4 ++-- worker/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 52eb06b..80da260 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -6,8 +6,8 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -RUN conda env update -n base -f environment.yml -RUN conda env create -n r -f environment_r.yml +RUN conda env update -f environment.yml +RUN conda env create -f environment_r.yml RUN jupyter labextension install \ @jupyter-widgets/jupyterlab-manager@0.38 \ diff --git a/worker/Dockerfile b/worker/Dockerfile index 3974844..12cea77 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -6,7 +6,7 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -RUN conda env create -n worker -f environment.yml +RUN conda env create -f environment.yml RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ From c4ca4753ebc97081e52bb35e8f1221ede35bc36a Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 19:32:53 -0700 Subject: [PATCH 007/156] another bug fix in conda build --- notebook/Dockerfile | 7 +++++-- worker/Dockerfile | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 80da260..cbe6538 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -6,8 +6,11 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -RUN conda env update -f environment.yml -RUN conda env create -f environment_r.yml + +COPY environment.yml /home/jovyan/environment.yml +RUN conda env update -f /home/jovyan/environment.yml +COPY environment_r.yml /home/jovyan/environment_r.yml +RUN conda env create -f /home/jovyan/environment_r.yml RUN jupyter labextension install \ @jupyter-widgets/jupyterlab-manager@0.38 \ diff --git a/worker/Dockerfile b/worker/Dockerfile index 12cea77..0c4f484 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -6,7 +6,8 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -RUN conda env create -f environment.yml +COPY environment.yml /opt/conda/environment.yml +RUN conda env create -f /opt/conda/environment.yml RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ From 321fd23d0bc64a0b3552a6d06906a485f795e952 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 20:19:16 -0700 Subject: [PATCH 008/156] unfreeze labextensions --- notebook/Dockerfile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index cbe6538..9c8197f 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -12,12 +12,17 @@ RUN conda env update -f /home/jovyan/environment.yml COPY environment_r.yml /home/jovyan/environment_r.yml RUN conda env create -f /home/jovyan/environment_r.yml -RUN jupyter labextension install \ - @jupyter-widgets/jupyterlab-manager@0.38 \ - @jupyterlab/hub-extension@0.12.0 \ - jupyterlab_bokeh@0.6.3 \ - @pyviz/jupyterlab_pyviz \ - dask-labextension +RUN jupyter labextension update --all && \ + jupyter labextension install \ + @jupyterlab/hub-extension \ + @jupyterlab/plotly-extension \ + @jupyter-widgets/jupyterlab-manager \ + @jupyter-widgets/jupyterlab-sidecar \ + @pyviz/jupyterlab_pyviz \ + dask-labextension \ + itk-jupyter-widgets \ + jupyterlab_bokeh \ + jupyter-leaflet RUN jupyter serverextension enable --py nbserverproxy --sys-prefix From c3313e9854df055241f6aa1cebc7712ec7887022 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 20:35:04 -0700 Subject: [PATCH 009/156] add ipyleaflet --- notebook/environment.yml | 1 + worker/environment.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/notebook/environment.yml b/notebook/environment.yml index dae9a39..5e49ce2 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -32,6 +32,7 @@ dependencies: - holoviews=1.12.5 - ipdb=0.12.2 - ipython=7.8.0 + - ipyleaflet=0.11.2 - ipywidgets=7.5.1 - jedi=0.15.1 - jupyter_client=5.3.3 diff --git a/worker/environment.yml b/worker/environment.yml index f024376..376be9f 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -32,6 +32,7 @@ dependencies: - holoviews=1.12.5 - ipdb=0.12.2 - ipython=7.8.0 + - ipyleaflet=0.11.2 - ipywidgets=7.5.1 - jedi=0.15.1 - jupyter_client=5.3.3 From 750fca095569f77ea2d4dbc3ed94b2bb7463c04a Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 21:02:54 -0700 Subject: [PATCH 010/156] add sidecar & leaflet, jupyter-server-proxy --- notebook/Dockerfile | 2 +- notebook/environment.yml | 6 ++++-- worker/environment.yml | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 9c8197f..2d3d867 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -24,7 +24,7 @@ RUN jupyter labextension update --all && \ jupyterlab_bokeh \ jupyter-leaflet -RUN jupyter serverextension enable --py nbserverproxy --sys-prefix +RUN jupyter serverextension enable --sys-prefix jupyter_server_proxy USER root COPY prepare.sh /usr/bin/prepare.sh diff --git a/notebook/environment.yml b/notebook/environment.yml index 5e49ce2..4bcc0ca 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -1,4 +1,4 @@ -name: notebook +name: base channels: - conda-forge dependencies: @@ -39,11 +39,11 @@ dependencies: - jupyter=1.0.0 - jupyterhub=1.0.0 - jupyterlab=1.1.4 + - jupyter-server-proxy - lz4=2.2.1 - matplotlib=3.1.1 - nb_conda_kernels=2.2 - nbconvert=5.6.0 - - nbserverproxy - ncurses - netcdf-fortran - netcdf4=1.5.1.2 @@ -53,6 +53,7 @@ dependencies: - numpy - pandas - pip + - plotly - pyshp=2.1.0 - python-blosc=1.8.1 - python-snappy=0.5.4 @@ -64,6 +65,7 @@ dependencies: - scipy=1.3.1 - seaborn=0.9.0 - shapely=1.6.4 + - sidecar - statsmodels=0.10.1 - toolz=0.10.0 - tornado=6.0.3 diff --git a/worker/environment.yml b/worker/environment.yml index 376be9f..58d8d34 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -49,6 +49,7 @@ dependencies: - numpy - pandas - pip + - plotly - pyshp=2.1.0 - python-blosc=1.8.1 - python-snappy=0.5.4 @@ -60,6 +61,7 @@ dependencies: - scipy=1.3.1 - seaborn=0.9.0 - shapely=1.6.4 + - sidecar - statsmodels=0.10.1 - toolz=0.10.0 - tornado=6.0.3 From 21b4d14bc057fbdbd60db6bb20751840fbc0508d Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 24 Sep 2019 22:23:10 -0700 Subject: [PATCH 011/156] oops. sidecar == pip --- notebook/environment.yml | 2 +- worker/environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notebook/environment.yml b/notebook/environment.yml index 4bcc0ca..e93cc2f 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -65,7 +65,6 @@ dependencies: - scipy=1.3.1 - seaborn=0.9.0 - shapely=1.6.4 - - sidecar - statsmodels=0.10.1 - toolz=0.10.0 - tornado=6.0.3 @@ -89,6 +88,7 @@ dependencies: - pytest-runner==4.2 - pytest==3.6.2 - Sphinx==1.7.5 + - sidecar - tox==3.1.1 - wget==3.2 - wheel==0.31.1 diff --git a/worker/environment.yml b/worker/environment.yml index 58d8d34..27d9276 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -61,7 +61,6 @@ dependencies: - scipy=1.3.1 - seaborn=0.9.0 - shapely=1.6.4 - - sidecar - statsmodels=0.10.1 - toolz=0.10.0 - tornado=6.0.3 @@ -84,6 +83,7 @@ dependencies: - pytest-runner==4.2 - pytest==3.6.2 - Sphinx==1.7.5 + - sidecar - tox==3.1.1 - wget==3.2 - wheel==0.31.1 From c2569a7f4a26ae5f136d141999663919274614d1 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Wed, 25 Sep 2019 16:35:52 -0700 Subject: [PATCH 012/156] install nodejs --- notebook/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/notebook/environment.yml b/notebook/environment.yml index e93cc2f..95dcaf3 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -47,6 +47,7 @@ dependencies: - ncurses - netcdf-fortran - netcdf4=1.5.1.2 + - nodejs - nose - numba=0.45.1 - numcodecs=0.6.3 From 05c5cdcc577b74c8530238e935e8876184264a6c Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Wed, 25 Sep 2019 17:17:40 -0700 Subject: [PATCH 013/156] add jupyter notebook, install jupyterlab serverextension --- notebook/Dockerfile | 2 ++ notebook/environment.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 2d3d867..acf027b 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -24,6 +24,8 @@ RUN jupyter labextension update --all && \ jupyterlab_bokeh \ jupyter-leaflet +RUN jupyter serverextension enable --py jupyterlab --sys-prefix +RUN jupyter serverextension enable --py jupyterlab --user RUN jupyter serverextension enable --sys-prefix jupyter_server_proxy USER root diff --git a/notebook/environment.yml b/notebook/environment.yml index 95dcaf3..fb8a634 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -49,6 +49,7 @@ dependencies: - netcdf4=1.5.1.2 - nodejs - nose + - notebook=6.0.1 - numba=0.45.1 - numcodecs=0.6.3 - numpy From 7bf5695f7686a3f3b4db6e4578e68c1ae56e07fb Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Wed, 25 Sep 2019 18:30:58 -0700 Subject: [PATCH 014/156] install/enable dask_labextension --- notebook/Dockerfile | 1 + notebook/environment.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index acf027b..f7792d6 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -27,6 +27,7 @@ RUN jupyter labextension update --all && \ RUN jupyter serverextension enable --py jupyterlab --sys-prefix RUN jupyter serverextension enable --py jupyterlab --user RUN jupyter serverextension enable --sys-prefix jupyter_server_proxy +RUN jupyter serverextension enable --py --sys-prefix dask_labextension USER root COPY prepare.sh /usr/bin/prepare.sh diff --git a/notebook/environment.yml b/notebook/environment.yml index fb8a634..595fd9e 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -81,6 +81,7 @@ dependencies: - bumpversion==0.5.3 - coverage==4.5.1 - git+git://github.com/dask/dask-kubernetes.git@295f06428119bc8e674a2e6a59cd97000fae69e8 + - dask_labextension - flake8==3.5.0 - google-cloud-storage==1.12.0 - kubernetes==6.0.0 From 058a5a7b36d187e660f5c1ac505bb5ea6549d40a Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 12:13:55 -0700 Subject: [PATCH 015/156] readme update --- README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e25130..39c90ab 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,21 @@ To update this file 4. Commit your changes 5. Tag your image with `python bump.py` 6. Push to github and make a pull request to master -7. If your build passes on Travis, we'll merge it and it will deploy to dockerhub +7. If your build passes on Travis, we'll merge it and it will deploy to dockerhub -Any questions please email jsimcock@rhg.com +Any questions please email mdelgado@rhg.com +# Cluster overview +* compute.rhg.com: flagship Rhodium compute cluster +* impactlab.rhg.org: flagship Climate Impact Lab compute cluster +Preemptable clusters: + +* coastal.rhg.com: pods in this cluster are cheaper but can disappear at any time. expect less stability, more bugs, popup errors, and lower bills. + +Testing clusters: + +* compute-test.rhg.com: staging deployment with stable users & user directories. This cluster should be used to beta-test deployments scheduled for the production servers in an environment similar to production. users should not expect their data here to be safe, but admins should make an effort to simulate production roll-outs and to ensure data/user safety in upgrading the cluster. admins should encourage production users to test their workflows on this cluster before a major production upgrade. +* testing.climate-kube.com: bleeding-edge test cluster. absolutely no guarantee of data/user/environment preservation. users should expect the entire cluster to be deleted at any point. +* test2.climate-kube.com: same purpose as testing.climate-kube.com, but another one to parallelize the madness. From a63dc2903cc0631dbb0b6768153c398ef175f3e7 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 12:21:50 -0700 Subject: [PATCH 016/156] upgrade rhg/cil tools & unpin pip packages --- notebook/Dockerfile | 5 +++-- notebook/environment.yml | 36 ++++++++++++++++++------------------ worker/environment.yml | 34 +++++++++++++++++----------------- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index f7792d6..c45dcd4 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -69,9 +69,10 @@ RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CO USER $NB_USER RUN pip install \ - rhg_compute_tools==0.1.5 \ + rhg_compute_tools==0.1.8 \ impactlab-tools==0.3.1 \ - climate-toolbox==0.1.4 \ + climate-toolbox==0.1.5 \ + parameterize_jobs==0.1.1 \ --no-cache-dir # update pip diff --git a/notebook/environment.yml b/notebook/environment.yml index 595fd9e..b1ce682 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -78,22 +78,22 @@ dependencies: - zeromq=4.3.2 - zict=1.0.0 - pip: - - bumpversion==0.5.3 - - coverage==4.5.1 - - git+git://github.com/dask/dask-kubernetes.git@295f06428119bc8e674a2e6a59cd97000fae69e8 - - dask_labextension - - flake8==3.5.0 - - google-cloud-storage==1.12.0 - - kubernetes==6.0.0 - - mapbox==0.16.2 - - py-noaa==0.2.2 - - pytest-cov==2.5.1 - - pytest-runner==4.2 - - pytest==3.6.2 - - Sphinx==1.7.5 + - bumpversion + - coverage + - dask-labextension + - dask-kubernetes==0.9.2 + - flake8 + - google-cloud-storage==1.20.0 + - kubernetes + - mapbox + - py-noaa + - pytest + - pytest-cov + - pytest-runner - sidecar - - tox==3.1.1 - - wget==3.2 - - wheel==0.31.1 - - xesmf==0.1.1 - - xlrd==1.1.0 + - Sphinx + - tox + - wget + - wheel + - xesmf + - xlrd diff --git a/worker/environment.yml b/worker/environment.yml index 27d9276..1102d7e 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -71,21 +71,21 @@ dependencies: - zeromq=4.3.2 - zict=1.0.0 - pip: - - bumpversion==0.5.3 - - coverage==4.5.1 - - git+git://github.com/dask/dask-kubernetes.git@295f06428119bc8e674a2e6a59cd97000fae69e8 - - flake8==3.5.0 - - google-cloud-storage==1.12.0 - - kubernetes==6.0.0 - - mapbox==0.16.2 - - py-noaa==0.2.2 - - pytest-cov==2.5.1 - - pytest-runner==4.2 - - pytest==3.6.2 - - Sphinx==1.7.5 + - bumpversion + - coverage + - dask-kubernetes==0.9.2 + - flake8 + - google-cloud-storage==1.20.0 + - kubernetes + - mapbox + - py-noaa + - pytest + - pytest-cov + - pytest-runner - sidecar - - tox==3.1.1 - - wget==3.2 - - wheel==0.31.1 - - xesmf==0.1.1 - - xlrd==1.1.0 + - Sphinx + - tox + - wget + - wheel + - xesmf + - xlrd From 726c8d6a5a46b88c02f58a54406b0f6f6395d2cc Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 12:25:14 -0700 Subject: [PATCH 017/156] add google cloud apis/sdks --- notebook/Dockerfile | 2 +- notebook/environment.yml | 5 ++++- worker/Dockerfile | 2 +- worker/environment.yml | 5 ++++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index c45dcd4..00cd162 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -43,7 +43,7 @@ COPY worker-template.yml /pre-home RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get update -y && apt-get install google-cloud-sdk -y + apt-get update -y && apt-get install google-cloud-sdk kubectl -y RUN export GCSFUSE_REPO=gcsfuse-xenial \ && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ diff --git a/notebook/environment.yml b/notebook/environment.yml index b1ce682..1b4ee80 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -83,7 +83,10 @@ dependencies: - dask-labextension - dask-kubernetes==0.9.2 - flake8 - - google-cloud-storage==1.20.0 + - google-api-core + - google-cloud + - google-cloud-container + - google-cloud-storage - kubernetes - mapbox - py-noaa diff --git a/worker/Dockerfile b/worker/Dockerfile index 75ef582..626edd7 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -13,7 +13,7 @@ RUN conda env create -f /opt/conda/environment.yml RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get update -y && apt-get install google-cloud-sdk -y + apt-get update -y && apt-get install google-cloud-sdk kubectl -y RUN export GCSFUSE_REPO=gcsfuse-xenial \ && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ diff --git a/worker/environment.yml b/worker/environment.yml index 1102d7e..12c71e6 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -75,7 +75,10 @@ dependencies: - coverage - dask-kubernetes==0.9.2 - flake8 - - google-cloud-storage==1.20.0 + - google-api-core + - google-cloud + - google-cloud-container + - google-cloud-storage - kubernetes - mapbox - py-noaa From 7d3c3d8c0b4a5657037ff807eb452354a0b5eca6 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 16:51:33 -0700 Subject: [PATCH 018/156] align packages, add package alignment tests --- maintenance_utilities/__init__.py | 0 maintenance_utilities/conda_tools.py | 318 +++++++++++++++++++++++++++ notebook/environment.yml | 1 + tests/test_package_alignment.py | 262 +++++++++------------- worker/environment.yml | 1 + 5 files changed, 426 insertions(+), 156 deletions(-) create mode 100644 maintenance_utilities/__init__.py create mode 100644 maintenance_utilities/conda_tools.py diff --git a/maintenance_utilities/__init__.py b/maintenance_utilities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/maintenance_utilities/conda_tools.py b/maintenance_utilities/conda_tools.py new file mode 100644 index 0000000..57e221b --- /dev/null +++ b/maintenance_utilities/conda_tools.py @@ -0,0 +1,318 @@ +''' +Utilities for parsing and managing conda environments & dependencies + +This module is used by the package alignment tests and (eventually) could be +used in tools to automatically find solutions for unified upgrades of multiple +connected environments. Additionally, we could develop tools to automate +environment pinning and other maintenance tasks. + +TODO: + +* account for upstream base notebook environment (ugh) +* build environments with "subdirs" specifying architecture of target servers + +''' + +import os +import re +import yaml +from conda.core.solve import Solver + +CONDA_ARGS = { + '-n': '--name', + '-f': '--file', + '-p': '--prefix', + '-c': '--channel', + '-S': '--satisfied-skip-solve', + '-m': '--mkdir', + '-C': '--use-index-cache', + '-k': '--insecure', + '-d': '--dry-run', + '-q': '--quiet', + '-v': '--verbose', + '-y': '--yes'} + + +def get_conda_solver( + filepath=None, + prefix=None, + channels=None, + subdirs=None, + specs_to_add=None, + existing_solver=None): + ''' + Initialize or update a conda environment solver from a file or spec + + Parameters + ---------- + filepath : str, optional + Path to a yaml file to parse and solve. If not provided, ``prefix`` is + required and any transactions must be defined in ``channels``, + ``subdirs``, and ``specs_to_add``. + prefix : str, optional + Name or path of the environment to be created/modified. Ignored if + ``filepath`` is provided; required if omitted. + channels: list, optional + Conda channels. Ignored if ``filepath`` is provided. + subdirs: tuple, optional + Architecture subdirs. Ignored if ``filepath`` is provided. + specs_to_add: list, optional + List of strings of package specs to add to the environment. Ignored if + ``filepath`` is provided. + existing_solver : conda.core.solve.Solver, optional + Optionally, upgrade a provided spec with the new dependencies provided + in an environment file. Default is to create a new environment. + + Returns + ------- + solver: conda.core.solve.Solver + + Examples + -------- + + Initializing with an environment file: + + .. code-block:: python + + >>> get_conda_solver('notebook/environment.yml') + + + Initializing with a spec + + .. code-block:: python + + >>> s = get_conda_solver( + ... prefix='vis', + ... channels=['pyviz', 'conda-forge'], + ... specs_to_add=['python=3.7', 'holoviews', 'bokeh>=1.2']) + ... + + Testing for unsatisfiable environments: + + .. code-block:: python + + >>> s = get_conda_solver( + ... prefix='base', + ... specs_to_add=['xarray>=0.13.0', 'python<3.0']) + ... + >>> s.solve_final_state() # doctest: +ELLIPSIS + ... + Traceback (most recent calls last): + ... + UnsatisfiableError: The following specifications were found + to be incompatible with the existing python installation in your environment: + + - xarray[version='>=0.13.0'] -> python[version='>=3.5.3'] + + If python is on the left-most side of the chain, that's the version you've asked for. + When python appears to the right, that indicates that the thing on the left is somehow + not available for the python version you've asked for. + Your current python version is (python[version='<3.0']). + + The following specifications were found to be incompatible with each other: + + - python[version='<3.0'] + + TODO + ---- + + * figure out how to solve for/align pip dependencies. currently these are + ignored + + ''' + + if filepath is not None: + with open(filepath, 'r') as f: + spec = yaml.safe_load(f) + else: + spec = { + 'prefix': prefix, + 'channels': channels if channels is not None else [], + 'subdirs': subdirs if subdirs is not None else [], + 'dependencies': specs_to_add if specs_to_add is not None else []} + + # exclude pip dependencies - these trip up the Solver + conda_packages = [ + d for d in spec.get('dependencies', []) if not isinstance(d, dict)] + + if existing_solver is None: + solver = Solver( + spec.get('name', 'base'), + channels=spec.get('channels', []), + specs_to_add=conda_packages) + else: + solver = Solver( + prefix=existing_solver.prefix, + channels=spec.get('channels', existing_solver.channels), + subdirs=spec.get('subdirs', existing_solver.subdirs), + specs_to_add=(list(existing_solver.specs_to_add) + conda_packages), + specs_to_remove=spec.get( + 'specs_to_remove', existing_solver.specs_to_remove)) + + return solver + +def parse_conda_create_command(env, args): + ''' + Convert a conda create/install/update command into a package spec + + TODO + ---- + + * accommodate ``conda remove`` command + + ''' + + if args[0].strip().lower() != 'conda': + return ('invalid', args) + + chaffe = ['conda', 'install', 'update', 'create', '-y', '--yes'] + install_args = [a for a in args if a.lower() not in chaffe] + if len(install_args) == 1 and install_args[0].split('=')[0] == 'conda': + # conda upgrade conda - take no action with API validation + # return {'upgrade': install_args[0].strip('"\''))} + pass + + spec = {'dependencies': []} + + get_install_args = iter(install_args) + + while True: + try: + arg = next(get_install_args) + except StopIteration: + break + + if arg.startswith('-'): + spec[CONDA_ARGS.get(arg, arg).lstrip('-')] = next(get_install_args) + + else: + pkgver = arg.strip('"\'') + spec['dependencies'].append(pkgver) + + spec['name'] = spec.get('name', env) + spec['channel'] = spec.get('channel', 'defaults') + + return spec + + +def get_conda_specs(dockerfile, conda_specs=None): + ''' + Scours a docker file for conda commands, and returns a spec for each env + + Parameters + ---------- + dockerfile: str + Path to a Dockerfile + conda_specs: dict, None + Existing conda specs from the Dockerfile build dependencies + + Returns + ------- + spec : dict + Dictionary of ``(env: solver)`` pairs, where the env is the name of + conda environments created by the dockerfile and solver is a + :py:class:`conda.core.solve.Solver` instance. + + ''' + + if conda_specs is None: + conda_specs = {} + + files_in_docker_scope = {} + + with open(dockerfile, 'r') as f: + docker_wd = os.path.dirname(dockerfile) + + line_getter = f + + in_conda = False + env = 'base' + + while True: + try: + line = next(f).split('#')[0].strip() + except StopIteration: + break + + args = re.split(r'\s+', line.rstrip('\\').strip()) + + # expand arg abbrevs + args = [CONDA_ARGS.get(a, a) for a in args] + + if args[0].upper() == 'COPY': + files_in_docker_scope[args[2]] = ( + os.path.join(docker_wd, args[1])) + elif ( + (args[0].upper() == 'RUN') + and (args[1].lower() in ['conda', 'source']) + and (args[2].lower() == 'activate')): + + env = args[3] + + elif ( + (args[0].upper() == 'RUN') + and (args[1].lower() == 'conda') + and (args[2].lower() in ['install', 'create'])): + + conda_spec = args[1:] + + if line.endswith('\\'): + in_conda = True + else: + spec = parse_conda_create_command(env, conda_spec) + solver = get_conda_solver( + prefix=spec['name'], + channels=spec.get('channels'), + subdirs=spec.get('subdirs'), + specs_to_add=spec.get('dependencies'), + existing_solver=conda_specs.get(spec['name'], None)) + + conda_specs[spec['name']] = solver + + elif ( + (args[0].upper() == 'RUN') + and (args[1].lower() == 'conda') + and (args[2].lower() == 'env') + and (args[3].lower() in ['create', 'update'])): + + if '--file' in args: + env_file = files_in_docker_scope[ + args[args.index('--file') + 1]] + + with open(env_file, 'r') as y: + env_spec = yaml.safe_load(y) + + solver = get_conda_solver( + env_file, + existing_solver=conda_specs.get( + env_spec['name'], None)) + + conda_specs[solver.prefix] = solver + + else: + raise ValueError( + "I can't parse this line: {}".format(line)) + + elif in_conda: + conda_spec += args + + if not line.endswith('\\'): + spec = parse_conda_create_command(env, conda_spec) + solver = get_conda_solver( + prefix=spec['name'], + channels=spec.get('channels'), + subdirs=spec.get('subdirs'), + specs_to_add=spec.get('dependencies'), + existing_solver=conda_specs.get(spec['name'], None)) + + conda_specs[spec['name']] = solver + in_conda = False + + return conda_specs + +def build_final_envs_for_multiple_docker_files(dockerfiles): + envs = {} + for dockerfile in dockerfiles: + envs.update(get_conda_specs(dockerfile, envs)) + + return envs diff --git a/notebook/environment.yml b/notebook/environment.yml index 1b4ee80..6f1d81b 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -30,6 +30,7 @@ dependencies: - git=2.23.0 - gitpython=3.0.2 - holoviews=1.12.5 + - icu=64.2 - ipdb=0.12.2 - ipython=7.8.0 - ipyleaflet=0.11.2 diff --git a/tests/test_package_alignment.py b/tests/test_package_alignment.py index 4e75457..cebf37e 100644 --- a/tests/test_package_alignment.py +++ b/tests/test_package_alignment.py @@ -1,177 +1,127 @@ +from __future__ import absolute_import + import os import re +import sys import pprint import conda.core.solve import pytest +if os.path.isdir('../maintenance_utilities'): + sys.path.append('../maintenance_utilities') +elif os.path.isdir('maintenance_utilities'): + sys.path.append('maintenance_utilities') + +import conda_tools + PKG_ROOT = os.path.dirname(os.path.dirname(__file__)) -IMAGES_TO_CHECK = [ - 'notebook', - 'worker', - ] - -PAIRINGS = { - 'notebook': { - 'base': ['worker'], - }, -} - -CONDA_ARGS = { - 'n': 'name', - 'p': 'prefix', - 'c': 'channel', - 'S': 'satisfied-skip-solve', - 'm': 'mkdir', - 'C': 'use-index-cache', - 'k': 'insecure', - 'd': 'dry-run', - 'q': 'quiet', - 'v': 'verbose', - 'y': 'yes'} - -def get_conda_specs(dockerfile): - with open(dockerfile, 'r') as f: - conda_specs = [] - - line_getter = f - - in_conda = False - env = 'base' - - while True: - try: - line = next(f).split('#')[0].strip() - except StopIteration: - break - - args = re.split(r'\s+', line.rstrip('\\').strip()) - - if ( - (args[0].upper() == 'RUN') - and (args[1].lower() in ['conda', 'source']) - and (args[2].lower() == 'activate')): - env = args[3] - - elif ( - (args[0].upper() == 'RUN') - and (args[1].lower() == 'conda') - and (args[2].lower() in ['install', 'create'])): - - conda_spec = args[1:] - - if line.endswith('\\'): - in_conda = True - else: - conda_specs.append((env, conda_spec)) - - elif in_conda: - conda_spec += args - - if not line.endswith('\\'): - conda_specs.append((env, conda_spec)) - in_conda = False - - return conda_specs - -def parse_conda_create_command(env, args): - if args[0].strip().lower() != 'conda': - return ('invalid', args) - - chaffe = ['conda', 'install', 'update', 'create', '-y', '--yes'] - install_args = [a for a in args if a.lower() not in chaffe] - if len(install_args) == 1 and install_args[0].split('=')[0] == 'conda': - return {'upgrade': tuple(install_args[0].strip('"\'').split('='))} - - spec = {'packages': {}} - - get_install_args = iter(install_args) - - while True: - try: - arg = next(get_install_args) - except StopIteration: - break - - if arg.startswith('-'): - a = arg.strip('-') - spec[CONDA_ARGS.get(a, a)] = next(get_install_args) - - else: - pkgver = arg.strip('"\'').split('=') - if len(pkgver) == 1: - pkgver.append(None) - spec['packages'].update({pkgver[0]: pkgver[1]}) - - spec['name'] = spec.get('name', env) - spec['channel'] = spec.get('channel', 'defaults') - - return spec - -def assert_pairing_match(env, base, penv, pairs): - for base_spec in base: - for pkg, ver in base_spec['packages'].items(): - for pair in pairs: - if pkg in pair.get('packages', {}): - pair_ver = pair['packages'][pkg] - test = (ver == pair_ver) - msg = ( - 'Package versions mis-aligned in env {}:{}' - '\n\tpackage {}: {} != {}' - .format(env, penv, pkg, ver, pair_ver)) - assert test, msg - -def assert_conda_spec_validity(img, spec): - for img_spec in spec: - if not 'packages' in img_spec: - continue - - # for s, cmd in img_spec.items(): - print( - 'Solving {} package spec {} with channel {}'.format( - img, - img_spec.get('name', 'base'), - img_spec.get('channel', 'defaults'))) +IMAGES_TO_CHECK = { + 'notebook': ['notebook/Dockerfile'], + 'worker': ['worker/Dockerfile'], + } + + +PAIRINGS = [ + ('notebook', 'base', 'worker', 'worker'), +] +''' +PAIRINGS - solver = conda.core.solve.Solver( - img_spec.get('name', 'base'), - channels=[img_spec.get('channel', 'defaults'), 'defaults'], - specs_to_add=tuple(map(lambda x: '='.join([i for i in x if i is not None]), img_spec['packages'].items()))) - - state = solver.solve_final_state() +A list of tuples of (notebook image, notebook env, worker image, worker env) +pairings. For each pairing, all packages will be tested against each other to +ensure we don't have any dependency conflicts. +''' @pytest.fixture(scope='module') def package_spec(): - specs = {} + ''' + Returns a nested dictionary of images and solved conda environments - for img_name in IMAGES_TO_CHECK: - files = os.listdir(os.path.join(PKG_ROOT, img_name)) - dockerfiles = [f for f in files if f.startswith('Dockerfile')] - specs[img_name] = [] - for dockerfile in dockerfiles: - fp = os.path.join(PKG_ROOT, img_name, dockerfile) - s = get_conda_specs(fp) - env_specs = [ - parse_conda_create_command(k, v) - for (k, v) in get_conda_specs(fp)] + For each image in IMAGES_TO_CHECK, loops over the provided dockerfiles + used to construct the image, and finds the final conda environments + created in the dockerfiles. For each environment, the package specs + are solved using the conda API and what is returned is a dictionary + of dictionaries of conda solutions, indexed by docker image and then + conda env within each image. - specs[img_name] += env_specs + This solution is time consuming and catches package specification errors, + dependency and build conflicts, channel specification errors, and other + things that can go wrong in the process of specifying environments. - yield specs + Once all image environments have been solved, the resulting nested dict + is yielded as a package spec that can be used in other unit tests, such + as in tests of notebook:worker env pairings. + + Examples + -------- + + For example, the following test would invoke the package_spec fixture + when run with pytest, and would loop over all images, all environments in + each image, and all packages within each environment, and test each to + ensure that any python versions encountered were greater than version 3.0 + .. code-block:: python -@pytest.mark.parametrize('base', PAIRINGS.keys()) -def test_package_pairing(package_spec, base): - pairs = PAIRINGS[base] - for env, paired in pairs.items(): - for p in paired: - nb_env = [s for s in package_spec[base] if 'name' in s and s['name'] == env] - assert_pairing_match(env, nb_env, p, package_spec[p]) + >>> from distutils.version import LooseVersion + + >>> def test_all_py3k(package_spec): + ... for img, envs in package_spec.items(): + ... for prefix, env in envs.items(): + ... for package in env: + ... if package.name == 'python': + ... assert LooseVersion(package.version) >= '3.0' + ... + + ''' + + specs = {} + + for img_name, dockerfiles in IMAGES_TO_CHECK.items(): + dockerfiles = [os.path.join(PKG_ROOT, fp) for fp in dockerfiles] + specs[img_name] = {} + + envs = ( + conda_tools + .build_final_envs_for_multiple_docker_files(dockerfiles)) + + for env, solver in envs.items(): + specs[img_name][env] = solver.solve_final_state() + + yield specs -@pytest.mark.parametrize('img', IMAGES_TO_CHECK) -def test_package_validity(package_spec, img): - spec = package_spec[img] - assert_conda_spec_validity(img, spec) +def assert_pairing_match(base_name, base_spec, paired_name, paired_spec): + failures = [] + for pkg in base_spec: + for pair in paired_spec: + if pkg.name == pair.name: + if pkg.version != pair.version: + failures.append((pkg, pair)) + + if len(failures) > 0: + raise ValueError( + 'Package versions mis-aligned in {} <> {} pairing:\n\t{}' + .format( + base_name, + paired_name, + '\n\t'.join([ + 'package {}: {} != {}' + .format(n.name, n.version, w.version) + for n, w in failures])) + ) + + +@pytest.mark.parametrize('pairing', PAIRINGS) +def test_package_pairing(package_spec, pairing): + notebook_image, notebook_env, worker_image, worker_env = pairing + notebook_spec = package_spec[notebook_image][notebook_env] + worker_spec = package_spec[worker_image][worker_env] + + assert_pairing_match( + notebook_image + ':' + notebook_env, + notebook_spec, + worker_image + ':' + worker_env, + worker_spec) diff --git a/worker/environment.yml b/worker/environment.yml index 12c71e6..a1b9879 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -30,6 +30,7 @@ dependencies: - git=2.23.0 - gitpython=3.0.2 - holoviews=1.12.5 + - icu=64.2 - ipdb=0.12.2 - ipython=7.8.0 - ipyleaflet=0.11.2 From cff3333cecda30b0337a5b4754f8554068f2355e Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 16:55:26 -0700 Subject: [PATCH 019/156] travis --- .travis.yml | 90 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index 656b31a..fecb220 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,37 +3,59 @@ sudo: required services: - docker -env: - matrix: - - IMAGE_NAME=notebook - - IMAGE_NAME=worker - -install: - - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" - - "rm notebook/worker-template.yml.bak" - - "cat notebook/worker-template.yml | grep image:" - - "cd $IMAGE_NAME" - - "docker pull rhodium/$IMAGE_NAME:dev" - - "docker build --pull --cache-from rhodium/$IMAGE_NAME:dev -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." - -script: - - docker images rhodium/$IMAGE_NAME:$TRAVIS_COMMIT - -deploy: - - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" - skip_cleanup: true - on: - branch: dev - - - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" - skip_cleanup: true - on: - branch: master - - - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" - skip_cleanup: true - on: - tags: true + + + + +jobs: + include: + - stage: alignment + language: python + python: + - 3.6 + script: + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + - conda info -a + - conda install pytest pytest-cov + - pytest + + - stage: docker + env: + matrix: + - IMAGE_NAME=notebook + - IMAGE_NAME=worker + + install: + - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" + - "rm notebook/worker-template.yml.bak" + - "cat notebook/worker-template.yml | grep image:" + - "cd $IMAGE_NAME" + - "docker pull rhodium/$IMAGE_NAME:dev" + - "docker build --pull --cache-from rhodium/$IMAGE_NAME:dev -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." + + script: + - docker images rhodium/$IMAGE_NAME:$TRAVIS_COMMIT + + deploy: + - provider: script + script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" + skip_cleanup: true + on: + branch: dev + + - provider: script + script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" + skip_cleanup: true + on: + branch: master + + - provider: script + script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" + skip_cleanup: true + on: + tags: true From 1724d6afa74208380121a94a176bf04614b07a6f Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 17:50:39 -0700 Subject: [PATCH 020/156] install yaml in alignment test env --- .travis.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index fecb220..a80e1e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,6 @@ sudo: required services: - docker - - - - jobs: include: - stage: alignment @@ -21,7 +17,7 @@ jobs: - conda config --set always_yes yes --set changeps1 no - conda update -q conda - conda info -a - - conda install pytest pytest-cov + - conda install pytest pytest-cov pyyaml - pytest - stage: docker @@ -29,7 +25,7 @@ jobs: matrix: - IMAGE_NAME=notebook - IMAGE_NAME=worker - + install: - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" - "rm notebook/worker-template.yml.bak" From b8dcf5da457ce7bf53eb9e403f9c31f3058d1a3d Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 17:54:50 -0700 Subject: [PATCH 021/156] drop icu? --- notebook/environment.yml | 1 - worker/environment.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/notebook/environment.yml b/notebook/environment.yml index 6f1d81b..1b4ee80 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -30,7 +30,6 @@ dependencies: - git=2.23.0 - gitpython=3.0.2 - holoviews=1.12.5 - - icu=64.2 - ipdb=0.12.2 - ipython=7.8.0 - ipyleaflet=0.11.2 diff --git a/worker/environment.yml b/worker/environment.yml index a1b9879..12c71e6 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -30,7 +30,6 @@ dependencies: - git=2.23.0 - gitpython=3.0.2 - holoviews=1.12.5 - - icu=64.2 - ipdb=0.12.2 - ipython=7.8.0 - ipyleaflet=0.11.2 From e3d6adc4613ebff5f4ba242ca94594c53d445c8c Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 17:55:55 -0700 Subject: [PATCH 022/156] but maybe still drop alignment tests for now --- .travis.yml | 28 ++++++++++++++-------------- notebook/Dockerfile | 2 +- worker/Dockerfile | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index a80e1e6..1aa2c88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,20 +5,20 @@ services: jobs: include: - - stage: alignment - language: python - python: - - 3.6 - script: - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - conda info -a - - conda install pytest pytest-cov pyyaml - - pytest + # - stage: alignment + # language: python + # python: + # - 3.6 + # script: + # - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + # - bash miniconda.sh -b -p $HOME/miniconda + # - export PATH="$HOME/miniconda/bin:$PATH" + # - hash -r + # - conda config --set always_yes yes --set changeps1 no + # - conda update -q conda + # - conda info -a + # - conda install pytest pytest-cov pyyaml + # - pytest - stage: docker env: diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 00cd162..b6cbf88 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -49,7 +49,7 @@ RUN export GCSFUSE_REPO=gcsfuse-xenial \ && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ && apt-get update \ - && apt-get install gcsfuse \ + && apt-get install gcsfuse -y \ && alias googlefuse=/usr/bin/gcsfuse USER root diff --git a/worker/Dockerfile b/worker/Dockerfile index 626edd7..5efead3 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -19,7 +19,7 @@ RUN export GCSFUSE_REPO=gcsfuse-xenial \ && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ && apt-get update \ - && apt-get install gcsfuse \ + && apt-get install gcsfuse -y \ && alias googlefuse=/usr/bin/gcsfuse # clean up From 9f2dedd66ab0c9524a54c82a2568c0ade4a94eee Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 18:15:19 -0700 Subject: [PATCH 023/156] re-introduce icu pin --- notebook/environment.yml | 1 + worker/environment.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/notebook/environment.yml b/notebook/environment.yml index 1b4ee80..6f1d81b 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -30,6 +30,7 @@ dependencies: - git=2.23.0 - gitpython=3.0.2 - holoviews=1.12.5 + - icu=64.2 - ipdb=0.12.2 - ipython=7.8.0 - ipyleaflet=0.11.2 diff --git a/worker/environment.yml b/worker/environment.yml index 12c71e6..a1b9879 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -30,6 +30,7 @@ dependencies: - git=2.23.0 - gitpython=3.0.2 - holoviews=1.12.5 + - icu=64.2 - ipdb=0.12.2 - ipython=7.8.0 - ipyleaflet=0.11.2 From 95152b559888a628dbe2c71587426b2da9456b5f Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 19:07:58 -0700 Subject: [PATCH 024/156] add octave notebook env & worker --- .travis.yml | 6 +-- notebook/Dockerfile | 12 +++-- notebook/environment_octave.yml | 76 ++++++++++++++++++++++++++++++ octave_worker/Dockerfile | 53 +++++++++++++++++++++ octave_worker/add_service_creds.py | 15 ++++++ octave_worker/environment.yml | 61 ++++++++++++++++++++++++ octave_worker/prepare.sh | 53 +++++++++++++++++++++ tests/test_package_alignment.py | 3 ++ worker/Dockerfile | 5 +- 9 files changed, 275 insertions(+), 9 deletions(-) create mode 100644 notebook/environment_octave.yml create mode 100644 octave_worker/Dockerfile create mode 100644 octave_worker/add_service_creds.py create mode 100644 octave_worker/environment.yml create mode 100644 octave_worker/prepare.sh diff --git a/.travis.yml b/.travis.yml index 1aa2c88..cc913dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,9 @@ jobs: - stage: docker env: - matrix: - - IMAGE_NAME=notebook - - IMAGE_NAME=worker + - IMAGE_NAME=notebook + - IMAGE_NAME=worker + - IMAGE_NAME=octave_worker install: - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" diff --git a/notebook/Dockerfile b/notebook/Dockerfile index b6cbf88..aae44e9 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -7,10 +7,14 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -COPY environment.yml /home/jovyan/environment.yml -RUN conda env update -f /home/jovyan/environment.yml -COPY environment_r.yml /home/jovyan/environment_r.yml -RUN conda env create -f /home/jovyan/environment_r.yml +RUN mkdir /opt/conda/specs + +COPY environment.yml /opt/conda/specs/environment.yml +RUN conda env update -f /opt/conda/specs/environment.yml +COPY environment_r.yml /opt/conda/specs/environment_r.yml +RUN conda env create -f /opt/conda/specs/environment_r.yml +COPY environment_octave.yml /opt/conda/specs/environment_octave.yml +RUN conda env create -f /opt/conda/specs/environment_octave.yml RUN jupyter labextension update --all && \ jupyter labextension install \ diff --git a/notebook/environment_octave.yml b/notebook/environment_octave.yml new file mode 100644 index 0000000..fdd90e5 --- /dev/null +++ b/notebook/environment_octave.yml @@ -0,0 +1,76 @@ +name: octave +channels: + - conda-forge +dependencies: + - octave=4.2.1 + - octave_kernel=0.31.0 + - portaudio=19.6.0 + - click + - cloudpickle=1.2.2 + - cytoolz=0.10.0 + - dask-core=2.4.0 + - dask-glm=0.2.0 + - dask-ml=1.0.0 + - dask=2.4.0 + - distributed=2.4.0 + - fastparquet=0.3.2 + - fusepy + - gcsfs=0.3.0 + - icu=64.2 + - ipdb=0.12.2 + - ipython=7.8.0 + - jedi=0.15.1 + - jupyter_client=5.3.3 + - jupyter=1.0.0 + - jupyterhub=1.0.0 + - jupyterlab=1.1.4 + - jupyter-server-proxy + - lz4=2.2.1 + - matplotlib=3.1.1 + - nb_conda_kernels=2.2 + - nbconvert=5.6.0 + - ncurses + - netcdf-fortran + - netcdf4=1.5.1.2 + - nodejs + - nose + - notebook=6.0.1 + - numba=0.45.1 + - numcodecs=0.6.3 + - numpy + - pandas + - pip + - python-blosc=1.8.1 + - python-snappy=0.5.4 + - PyYAML + - rasterio=1.0.28 + - scipy=1.3.1 + - statsmodels=0.10.1 + - toolz=0.10.0 + - tornado=6.0.3 + - tqdm=4.36.1 + - urllib3=1.25.6 + - widgetsnbextension=3 + - xarray=0.13.0 + - zarr=2.3.2 + - zeromq=4.3.2 + - zict=1.0.0 + - pip: + - bumpversion + - coverage + - dask-labextension + - dask-kubernetes==0.9.2 + - flake8 + - google-api-core + - google-cloud + - google-cloud-container + - google-cloud-storage + - kubernetes + - pytest + - pytest-cov + - pytest-runner + - tox + - wget + - wheel + - xesmf + - xlrd diff --git a/octave_worker/Dockerfile b/octave_worker/Dockerfile new file mode 100644 index 0000000..1212d31 --- /dev/null +++ b/octave_worker/Dockerfile @@ -0,0 +1,53 @@ +FROM climateimpactlab/miniconda3-gfortran-netcdf:v1.0.2 + +# build conda + +USER $NB_USER + +RUN conda config --add channels conda-forge +RUN conda update --yes conda + +RUN mkdir /opt/conda/specs +COPY environment.yml /opt/conda/specs/environment.yml +RUN conda env create -f /opt/conda/specs/environment.yml + +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update -y && apt-get install google-cloud-sdk kubectl -y + +RUN export GCSFUSE_REPO=gcsfuse-xenial \ + && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ + && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ + && apt-get update \ + && apt-get install gcsfuse -y \ + && alias googlefuse=/usr/bin/gcsfuse + +# clean up +RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN conda clean -tipsy + +ENV OMP_NUM_THREADS=1 +ENV DASK_TICK_MAXIMUM_DELAY=5s + +USER root +RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy +RUN chmod +x /usr/bin/cloud_sql_proxy + +USER root +COPY prepare.sh /usr/bin/prepare.sh +COPY add_service_creds.py /usr/bin/add_service_creds.py +RUN chmod +x /usr/bin/prepare.sh +RUN mkdir /opt/app +RUN mkdir /gcs + +RUN /opt/conda/envs/worker/bin/pip install \ + rhg_compute_tools==0.1.0 \ + impactlab-tools==0.3.1 \ + climate-toolbox==0.1.4 \ + --no-cache-dir + +# update pip +RUN /opt/conda/envs/worker/bin/pip install --upgrade pip + +ENTRYPOINT ["/usr/local/bin/dumb-init", "/usr/bin/prepare.sh"] diff --git a/octave_worker/add_service_creds.py b/octave_worker/add_service_creds.py new file mode 100644 index 0000000..f8c0385 --- /dev/null +++ b/octave_worker/add_service_creds.py @@ -0,0 +1,15 @@ +import json, os + + +def create_service_cred_files(): + + with open("/opt/gcsfuse_token_strings.json", 'r') as f: + creds = json.load(f) + + for k, v in creds.items(): + with open('/opt/gcsfuse_tokens/{}.json'.format(k), 'w+') as f: + f.write(json.dumps(v)) + + +if __name__ == '__main__': + create_service_cred_files() diff --git a/octave_worker/environment.yml b/octave_worker/environment.yml new file mode 100644 index 0000000..661249a --- /dev/null +++ b/octave_worker/environment.yml @@ -0,0 +1,61 @@ +name: worker +channels: + - conda-forge +dependencies: + - octave=4.2.1 + - octave_kernel=0.31.0 + - portaudio=19.6.0 + - click + - cloudpickle=1.2.2 + - cytoolz=0.10.0 + - dask-core=2.4.0 + - dask-glm=0.2.0 + - dask-ml=1.0.0 + - dask=2.4.0 + - distributed=2.4.0 + - fastparquet=0.3.2 + - fusepy + - gcsfs=0.3.0 + - icu=64.2 + - ipdb=0.12.2 + - ipython=7.8.0 + - jedi=0.15.1 + - jupyter_client=5.3.3 + - jupyter=1.0.0 + - lz4=2.2.1 + - matplotlib=3.1.1 + - nb_conda_kernels=2.2 + - ncurses + - netcdf-fortran + - netcdf4=1.5.1.2 + - nose + - numba=0.45.1 + - numcodecs=0.6.3 + - numpy + - pandas + - pip + - python-blosc=1.8.1 + - python-snappy=0.5.4 + - PyYAML + - rasterio=1.0.28 + - scipy=1.3.1 + - statsmodels=0.10.1 + - toolz=0.10.0 + - tornado=6.0.3 + - tqdm=4.36.1 + - urllib3=1.25.6 + - xarray=0.13.0 + - zarr=2.3.2 + - zeromq=4.3.2 + - zict=1.0.0 + - pip: + - dask-kubernetes==0.9.2 + - google-api-core + - google-cloud + - google-cloud-container + - google-cloud-storage + - kubernetes + - wget + - wheel + - xesmf + - xlrd diff --git a/octave_worker/prepare.sh b/octave_worker/prepare.sh new file mode 100644 index 0000000..6bf35b6 --- /dev/null +++ b/octave_worker/prepare.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -x + +if [[ -e "/opt/app/environment.yml" ]]; then + echo "environment.yml found. Installing packages" + /opt/conda/bin/conda env update -n worker -f /opt/app/environment.yml +else + echo "no environment.yml" +fi + +if [[ "$EXTRA_CONDA_PACKAGES" ]]; then + echo "EXTRA_CONDA_PACKAGES environment variable found. Installing." + /opt/conda/bin/conda install -n worker --yes $EXTRA_CONDA_PACKAGES +fi + +if [[ "$EXTRA_PIP_PACKAGES" ]]; then + echo "EXTRA_PIP_PACKAGES environment variable found. Installing". + /opt/conda/envs/worker/bin/pip install $EXTRA_PIP_PACKAGES +fi + +if [[ "$GCSFUSE_TOKENS" ]]; then + echo "$GCSFUSE_TOKENS" > /opt/gcsfuse_token_strings.json; + + mkdir -p /opt/gcsfuse_tokens/; + + python /usr/bin/add_service_creds.py; + + for f in /opt/gcsfuse_tokens/*.json; + do + bucket=$(basename ${f/.json/}); + if ! grep -q "/gcs/${bucket}" /proc/mounts; then + echo "Mounting $bucket to /gcs/$bucket"; + mkdir -p "/gcs/${bucket}"; + /usr/bin/gcsfuse --key-file="$f" "${bucket}" "/gcs/${bucket}"; + fi; + done +fi + +if [[ "$SQL_TOKEN" ]]; then + if [[ "$SQL_INSTANCE" ]]; then + echo "Starting SQL proxy connection to $SQL_INSTANCE"; + echo "$SQL_TOKEN" > /opt/sql_token_string.json; + /usr/bin/cloud_sql_proxy -instances=$SQL_INSTANCE -credential_file=/opt/sql_token_string.json & + fi; +fi + +if [[ "$GCLOUD_DEFAULT_TOKEN_FILE" ]]; then + gcloud auth activate-service-account --key-file $GCLOUD_DEFAULT_TOKEN_FILE; +fi + +# Run extra commands +source activate worker && $@ diff --git a/tests/test_package_alignment.py b/tests/test_package_alignment.py index cebf37e..ffdb09a 100644 --- a/tests/test_package_alignment.py +++ b/tests/test_package_alignment.py @@ -20,11 +20,13 @@ IMAGES_TO_CHECK = { 'notebook': ['notebook/Dockerfile'], 'worker': ['worker/Dockerfile'], + 'octave': ['octave/Dockerfile'], } PAIRINGS = [ ('notebook', 'base', 'worker', 'worker'), + ('notebook', 'octave', 'worker', 'octave'), ] ''' PAIRINGS @@ -32,6 +34,7 @@ A list of tuples of (notebook image, notebook env, worker image, worker env) pairings. For each pairing, all packages will be tested against each other to ensure we don't have any dependency conflicts. + ''' diff --git a/worker/Dockerfile b/worker/Dockerfile index 5efead3..1212d31 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -7,8 +7,9 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -COPY environment.yml /opt/conda/environment.yml -RUN conda env create -f /opt/conda/environment.yml +RUN mkdir /opt/conda/specs +COPY environment.yml /opt/conda/specs/environment.yml +RUN conda env create -f /opt/conda/specs/environment.yml RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ From c92a750369f702b17d645c4d0949c48330e9f0f7 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 19:08:34 -0700 Subject: [PATCH 025/156] fix build matrix --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1aa2c88..080c016 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,8 @@ jobs: - stage: docker env: - matrix: - - IMAGE_NAME=notebook - - IMAGE_NAME=worker + - IMAGE_NAME=notebook + - IMAGE_NAME=worker install: - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" From b3528caed50f37175575db44ab08d524a9156b4a Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 19:20:32 -0700 Subject: [PATCH 026/156] fix build matrix --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 080c016..2fab258 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,10 @@ sudo: required services: - docker +env: +- IMAGE_NAME=notebook +- IMAGE_NAME=worker + jobs: include: # - stage: alignment @@ -21,9 +25,6 @@ jobs: # - pytest - stage: docker - env: - - IMAGE_NAME=notebook - - IMAGE_NAME=worker install: - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" From 84a8753f6797fb31b6586b9c70e052f90caa3c07 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 19:25:06 -0700 Subject: [PATCH 027/156] fix build matrix --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2fab258..d6eeb7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ jobs: # - conda install pytest pytest-cov pyyaml # - pytest + matrix: - stage: docker install: From afd2cfc472fec11907ad3e7694ef30b55abf1d01 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 19:35:01 -0700 Subject: [PATCH 028/156] fix build matrix --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d6eeb7b..1ee8545 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ env: - IMAGE_NAME=notebook - IMAGE_NAME=worker -jobs: - include: +stages: + # - stage: alignment # language: python # python: @@ -24,8 +24,7 @@ jobs: # - conda install pytest pytest-cov pyyaml # - pytest - matrix: - - stage: docker + - name: docker install: - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" From d09e0102e38c325ce65f7ed4a5d8171aac942102 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 19:37:57 -0700 Subject: [PATCH 029/156] fix build matrix --- .travis.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1ee8545..2553190 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,24 +7,9 @@ env: - IMAGE_NAME=notebook - IMAGE_NAME=worker -stages: - - # - stage: alignment - # language: python - # python: - # - 3.6 - # script: - # - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - # - bash miniconda.sh -b -p $HOME/miniconda - # - export PATH="$HOME/miniconda/bin:$PATH" - # - hash -r - # - conda config --set always_yes yes --set changeps1 no - # - conda update -q conda - # - conda info -a - # - conda install pytest pytest-cov pyyaml - # - pytest - - - name: docker +jobs: + include: + - stage: docker install: - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" @@ -55,3 +40,18 @@ stages: skip_cleanup: true on: tags: true + + # - stage: alignment + # language: python + # python: + # - 3.6 + # script: + # - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + # - bash miniconda.sh -b -p $HOME/miniconda + # - export PATH="$HOME/miniconda/bin:$PATH" + # - hash -r + # - conda config --set always_yes yes --set changeps1 no + # - conda update -q conda + # - conda info -a + # - conda install pytest pytest-cov pyyaml + # - pytest From 35960ea70578a0dc36202ec18a05d1957886be16 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 19:40:42 -0700 Subject: [PATCH 030/156] fix build matrix --- .travis.yml | 62 +++++++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2553190..7e57d90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,39 +7,35 @@ env: - IMAGE_NAME=notebook - IMAGE_NAME=worker -jobs: - include: - - stage: docker - - install: - - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" - - "rm notebook/worker-template.yml.bak" - - "cat notebook/worker-template.yml | grep image:" - - "cd $IMAGE_NAME" - - "docker pull rhodium/$IMAGE_NAME:dev" - - "docker build --pull --cache-from rhodium/$IMAGE_NAME:dev -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." - - script: - - docker images rhodium/$IMAGE_NAME:$TRAVIS_COMMIT - - deploy: - - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" - skip_cleanup: true - on: - branch: dev - - - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" - skip_cleanup: true - on: - branch: master - - - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" - skip_cleanup: true - on: - tags: true +install: +- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" +- "rm notebook/worker-template.yml.bak" +- "cat notebook/worker-template.yml | grep image:" +- "cd $IMAGE_NAME" +- "docker pull rhodium/$IMAGE_NAME:dev" +- "docker build --pull --cache-from rhodium/$IMAGE_NAME:dev -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." + +script: +- docker images rhodium/$IMAGE_NAME:$TRAVIS_COMMIT + +deploy: +- provider: script + script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" + skip_cleanup: true + on: + branch: dev + +- provider: script + script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" + skip_cleanup: true + on: + branch: master + +- provider: script + script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" + skip_cleanup: true + on: + tags: true # - stage: alignment # language: python From e2c642ce937344ca04c841a53586ecca7f4ad810 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 20:35:50 -0700 Subject: [PATCH 031/156] drop xemf --- notebook/environment_octave.yml | 2 -- octave_worker/environment.yml | 1 - worker/environment.yml | 1 - 3 files changed, 4 deletions(-) diff --git a/notebook/environment_octave.yml b/notebook/environment_octave.yml index fdd90e5..cdaba3e 100644 --- a/notebook/environment_octave.yml +++ b/notebook/environment_octave.yml @@ -72,5 +72,3 @@ dependencies: - tox - wget - wheel - - xesmf - - xlrd diff --git a/octave_worker/environment.yml b/octave_worker/environment.yml index 661249a..1743ca9 100644 --- a/octave_worker/environment.yml +++ b/octave_worker/environment.yml @@ -57,5 +57,4 @@ dependencies: - kubernetes - wget - wheel - - xesmf - xlrd diff --git a/worker/environment.yml b/worker/environment.yml index a1b9879..7803ba2 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -91,5 +91,4 @@ dependencies: - tox - wget - wheel - - xesmf - xlrd From a130b899d3fb80f890f5c09d913f97d1a747cd19 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Thu, 26 Sep 2019 23:19:23 -0700 Subject: [PATCH 032/156] rename octave_worker --> octave-worker --- .travis.yml | 2 +- {octave_worker => octave-worker}/Dockerfile | 0 {octave_worker => octave-worker}/add_service_creds.py | 0 {octave_worker => octave-worker}/environment.yml | 0 {octave_worker => octave-worker}/prepare.sh | 0 5 files changed, 1 insertion(+), 1 deletion(-) rename {octave_worker => octave-worker}/Dockerfile (100%) rename {octave_worker => octave-worker}/add_service_creds.py (100%) rename {octave_worker => octave-worker}/environment.yml (100%) rename {octave_worker => octave-worker}/prepare.sh (100%) diff --git a/.travis.yml b/.travis.yml index 071a235..0119ddb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ services: env: - IMAGE_NAME=notebook - IMAGE_NAME=worker -- IMAGE_NAME=octave_worker +- IMAGE_NAME=octave-worker install: - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" diff --git a/octave_worker/Dockerfile b/octave-worker/Dockerfile similarity index 100% rename from octave_worker/Dockerfile rename to octave-worker/Dockerfile diff --git a/octave_worker/add_service_creds.py b/octave-worker/add_service_creds.py similarity index 100% rename from octave_worker/add_service_creds.py rename to octave-worker/add_service_creds.py diff --git a/octave_worker/environment.yml b/octave-worker/environment.yml similarity index 100% rename from octave_worker/environment.yml rename to octave-worker/environment.yml diff --git a/octave_worker/prepare.sh b/octave-worker/prepare.sh similarity index 100% rename from octave_worker/prepare.sh rename to octave-worker/prepare.sh From dd150d449677801c0a409e2d9da439ad7b79e881 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 27 Sep 2019 08:39:24 -0700 Subject: [PATCH 033/156] update package alignment for octave worker --- tests/test_package_alignment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_package_alignment.py b/tests/test_package_alignment.py index ffdb09a..e9b6482 100644 --- a/tests/test_package_alignment.py +++ b/tests/test_package_alignment.py @@ -20,13 +20,13 @@ IMAGES_TO_CHECK = { 'notebook': ['notebook/Dockerfile'], 'worker': ['worker/Dockerfile'], - 'octave': ['octave/Dockerfile'], + 'octave': ['octave-worker/Dockerfile'], } PAIRINGS = [ ('notebook', 'base', 'worker', 'worker'), - ('notebook', 'octave', 'worker', 'octave'), + ('notebook', 'octave', 'octave-worker', 'octave'), ] ''' PAIRINGS From 471350acc90ac496772023cace5e276356070b72 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 27 Sep 2019 10:51:20 -0700 Subject: [PATCH 034/156] add oct2py --- notebook/environment_octave.yml | 1 + octave-worker/environment.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/notebook/environment_octave.yml b/notebook/environment_octave.yml index cdaba3e..84b24ca 100644 --- a/notebook/environment_octave.yml +++ b/notebook/environment_octave.yml @@ -4,6 +4,7 @@ channels: dependencies: - octave=4.2.1 - octave_kernel=0.31.0 + - oct2py=5.0.4 - portaudio=19.6.0 - click - cloudpickle=1.2.2 diff --git a/octave-worker/environment.yml b/octave-worker/environment.yml index 1743ca9..f0d6319 100644 --- a/octave-worker/environment.yml +++ b/octave-worker/environment.yml @@ -4,6 +4,7 @@ channels: dependencies: - octave=4.2.1 - octave_kernel=0.31.0 + - oct2py=5.0.4 - portaudio=19.6.0 - click - cloudpickle=1.2.2 From c66b6d910bccff118e37534c2414569ea3b6b023 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 27 Sep 2019 12:44:55 -0700 Subject: [PATCH 035/156] install & upgrade rhg/cil packages --- notebook/Dockerfile | 24 ++++++++++++++++-------- octave-worker/Dockerfile | 9 +++++---- worker/Dockerfile | 9 +++++---- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index aae44e9..f524e69 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -72,15 +72,23 @@ RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers USER $NB_USER -RUN pip install \ - rhg_compute_tools==0.1.8 \ - impactlab-tools==0.3.1 \ - climate-toolbox==0.1.5 \ - parameterize_jobs==0.1.1 \ - --no-cache-dir - # update pip -RUN pip install --upgrade pip +RUN /opt/conda/bin/pip install --upgrade pip +RUN /opt/conda/envs/octave/bin/pip install --upgrade pip + +RUN /opt/conda/bin/pip install \ + rhg_compute_tools==0.1.8 \ + impactlab-tools==0.3.1 \ + climate-toolbox==0.1.5 \ + parameterize_jobs==0.1.1 \ + --no-cache-dir + +RUN /opt/conda/envs/octave/bin/pip install \ + rhg_compute_tools==0.1.8 \ + impactlab-tools==0.3.1 \ + climate-toolbox==0.1.5 \ + parameterize_jobs==0.1.1 \ + --no-cache-dir WORKDIR $HOME diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile index 1212d31..5ccd4e0 100644 --- a/octave-worker/Dockerfile +++ b/octave-worker/Dockerfile @@ -42,10 +42,11 @@ RUN mkdir /opt/app RUN mkdir /gcs RUN /opt/conda/envs/worker/bin/pip install \ - rhg_compute_tools==0.1.0 \ - impactlab-tools==0.3.1 \ - climate-toolbox==0.1.4 \ - --no-cache-dir + rhg_compute_tools==0.1.8 \ + impactlab-tools==0.3.1 \ + climate-toolbox==0.1.5 \ + parameterize_jobs==0.1.1 \ + --no-cache-dir # update pip RUN /opt/conda/envs/worker/bin/pip install --upgrade pip diff --git a/worker/Dockerfile b/worker/Dockerfile index 1212d31..5ccd4e0 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -42,10 +42,11 @@ RUN mkdir /opt/app RUN mkdir /gcs RUN /opt/conda/envs/worker/bin/pip install \ - rhg_compute_tools==0.1.0 \ - impactlab-tools==0.3.1 \ - climate-toolbox==0.1.4 \ - --no-cache-dir + rhg_compute_tools==0.1.8 \ + impactlab-tools==0.3.1 \ + climate-toolbox==0.1.5 \ + parameterize_jobs==0.1.1 \ + --no-cache-dir # update pip RUN /opt/conda/envs/worker/bin/pip install --upgrade pip From b3ece1ff3953b4fd7c461bb50f3e19a3606d4024 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 27 Sep 2019 12:46:32 -0700 Subject: [PATCH 036/156] bug fix in alignment test --- tests/test_package_alignment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_package_alignment.py b/tests/test_package_alignment.py index e9b6482..d0252e2 100644 --- a/tests/test_package_alignment.py +++ b/tests/test_package_alignment.py @@ -26,7 +26,7 @@ PAIRINGS = [ ('notebook', 'base', 'worker', 'worker'), - ('notebook', 'octave', 'octave-worker', 'octave'), + ('notebook', 'octave', 'octave-worker', 'worker'), ] ''' PAIRINGS From b7a9794459ba2c92c50525861854f3dcc5cf34aa Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 27 Sep 2019 14:05:37 -0700 Subject: [PATCH 037/156] add vertical ruler --- notebook/Dockerfile | 2 ++ notebook/overrides.json | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 notebook/overrides.json diff --git a/notebook/Dockerfile b/notebook/Dockerfile index b6cbf88..85f6fbe 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -80,5 +80,7 @@ RUN pip install --upgrade pip WORKDIR $HOME +COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json + ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] CMD ["start.sh jupyter lab"] diff --git a/notebook/overrides.json b/notebook/overrides.json new file mode 100644 index 0000000..82b1316 --- /dev/null +++ b/notebook/overrides.json @@ -0,0 +1,7 @@ +{ + "@jupyterlab/fileeditor-extension:plugin": { + "editorConfig": { + "rulers": [79] + } + } +} From d035ca17205596f70ea5bbc969bbcdda8d264a07 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 27 Sep 2019 16:52:37 -0700 Subject: [PATCH 038/156] temporary travis change --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7e57d90..e1e1dc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,10 +20,8 @@ script: deploy: - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" + script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:$TRAVIS_BRANCH && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" skip_cleanup: true - on: - branch: dev - provider: script script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" From 2916f0134607b03e4849f2889379cd2ca0861936 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 27 Sep 2019 17:06:21 -0700 Subject: [PATCH 039/156] try travis build dev tag --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index e1e1dc0..956791e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,9 @@ deploy: - provider: script script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:$TRAVIS_BRANCH && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" skip_cleanup: true + branches: + only: + - /^dev/ - provider: script script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" From bbac8998c0721f801a8897d96a223da826420f2a Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 27 Sep 2019 17:29:26 -0700 Subject: [PATCH 040/156] add libgl1-mesa-glx & upgrade gnuplot --- notebook/Dockerfile | 4 ++++ octave-worker/Dockerfile | 4 ++++ worker/Dockerfile | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index f524e69..0608cdc 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,5 +1,9 @@ FROM climateimpactlab/gfortran-netcdf-notebook:v1.0.3 +RUN apt-get update \ + && apt-get install libgl1-mesa-glx -y \ + && apt-get upgrade gnuplot -y + # build conda USER $NB_USER diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile index 5ccd4e0..d81ef53 100644 --- a/octave-worker/Dockerfile +++ b/octave-worker/Dockerfile @@ -1,5 +1,9 @@ FROM climateimpactlab/miniconda3-gfortran-netcdf:v1.0.2 +RUN apt-get update \ + && apt-get install libgl1-mesa-glx -y \ + && apt-get upgrade gnuplot -y + # build conda USER $NB_USER diff --git a/worker/Dockerfile b/worker/Dockerfile index 5ccd4e0..d81ef53 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -1,5 +1,9 @@ FROM climateimpactlab/miniconda3-gfortran-netcdf:v1.0.2 +RUN apt-get update \ + && apt-get install libgl1-mesa-glx -y \ + && apt-get upgrade gnuplot -y + # build conda USER $NB_USER From 48889f8c8a1aba556b0f5de31f7d85a7c370c4af Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 27 Sep 2019 17:30:52 -0700 Subject: [PATCH 041/156] retry fix travis --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 956791e..b38b7e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,8 @@ deploy: - provider: script script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:$TRAVIS_BRANCH && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" skip_cleanup: true - branches: - only: - - /^dev/ + on: + all_branches: true - provider: script script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" From 0fecacf3196c5557fff93449fb54ef2f3be1fff6 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 27 Sep 2019 19:45:06 -0700 Subject: [PATCH 042/156] add vertical ruler to notebook too --- notebook/overrides.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/notebook/overrides.json b/notebook/overrides.json index 82b1316..a971839 100644 --- a/notebook/overrides.json +++ b/notebook/overrides.json @@ -3,5 +3,10 @@ "editorConfig": { "rulers": [79] } + }, + "@jupyterlab/notebook-extension:tracker": { + "codeCellConfig": { + "rulers": [79] + } } } From cc85202691e667087fb26c1f06b79f7d876a56f9 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 28 Sep 2019 13:57:25 -0700 Subject: [PATCH 043/156] update travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b38b7e8..94daf86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ deploy: skip_cleanup: true on: all_branches: true + condition: $TRAVIS_BRANCH =~ ^dev - provider: script script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" From 8a9ab8b05b82b5d74712dba46065f60f0f0b1004 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 4 Oct 2019 22:45:00 -0700 Subject: [PATCH 044/156] try unpinning everything --- base_environment.yml | 85 +++++++++++++++++++++++++++ conda_environment.yml | 57 ------------------ notebook/Dockerfile | 22 +++---- notebook/environment.yml | 108 ++++------------------------------- notebook/requirements.txt | 13 ----- notebook/worker-template.yml | 2 +- worker/Dockerfile | 43 ++++++++++---- worker/environment.yml | 93 +----------------------------- 8 files changed, 141 insertions(+), 282 deletions(-) create mode 100644 base_environment.yml delete mode 100644 conda_environment.yml delete mode 100644 notebook/requirements.txt diff --git a/base_environment.yml b/base_environment.yml new file mode 100644 index 0000000..cf8144e --- /dev/null +++ b/base_environment.yml @@ -0,0 +1,85 @@ +name: base +channels: + - conda-forge +dependencies: + - beautifulsoup4 + - bokeh + - bqplot + - bumpversion + - cartopy + - click + - cloudpickle + - coverage + - cytoolz + - dask + - dask-glm + - dask-kubernetes + - dask-ml + - datashader + - distributed + - esmf + - esmpy + - fastparquet + - fiona + - flake8 + - fusepy + - gcsfs + - gdal + - geoalchemy2 + - geopy + - geotiff + - geoviews + - git + - gitpython + - google-cloud-container + - google-cloud-storage + - holoviews + - icu + - ipdb + - ipyleaflet + - jedi + - kubernetes + - lz4=2.2.1 + - matplotlib + - nb_conda_kernels + - ncurses + - netcdf-fortran + - netcdf4 + - numba + - numcodecs + - pandas + - pip + - plotly + - pyshp + - python-blosc + - python-snapp + - pyviz_comms + - PyYAML + - rasterio + - scikit-image + - scikit-learn + - scipy + - seaborn + - shapely + - sidecar + - statsmodels + - toolz + - tornado + - tox + - tqdm + - urllib3 + - wget + - wheel + - xarray + - xesmf + - xlrd + - zarr + - zeromq + - zict + - pip: + - mapbox + - py-noaa + - rhg_compute_tools + - impactlab-tools + - climate-toolbox + - parameterize_jobs diff --git a/conda_environment.yml b/conda_environment.yml deleted file mode 100644 index c54efd5..0000000 --- a/conda_environment.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: spatial -channels: - - defaults - - conda-forge -dependencies: - - python=3.6 - - bokeh=0.12.14 - - cytoolz - - datashader - - dask=0.17.2 - - gdal=2.2.4 - - esmpy - - zarr - - distributed=1.21.5 - - fastparquet - - git - - ipywidgets - - jupyterlab - - ipython - - jupyter - - holoviews - - lz4=1.1.0 - - matplotlib - - nb_conda_kernels - - netcdf4 - - nomkl - - numba=0.37.0 - - numcodecs - - numpy=1.14.2 - - pandas - - python-blosc=1.4.4 - - scipy - - scikit-image - - tornado - - xarray=0.10.7 - - zict - - rasterio - - cartopy - - conda-forge/label/dev::geopandas - - pip: - - wget - - google-cloud==0.32.0 - - google-cloud-storage - - gsutil - - fusepy - - click - - jedi - - kubernetes - - pyasn1 - - click - - urllib3 - - xesmf - - daskernetes==0.1.3 - - git+https://github.com/dask/dask-kubernetes@5ba08f714ef38e585e9f2038b6be530c578b96dd - - git+https://github.com/ioam/holoviews@3f015c0a531f54518abbfecffaac72a7b3554ed3 - - git+https://github.com/dask/gcsfs@2fbdc27e838a531ada080886ae778cb370ae48b8 - - git+https://github.com/jupyterhub/nbserverproxy diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 85f6fbe..4283328 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,4 +1,4 @@ -FROM climateimpactlab/gfortran-netcdf-notebook:v1.0.3 +FROM jupyter/base-notebook:python-3.7.3 # build conda @@ -7,8 +7,18 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda +# install cythonized version of geopandas +RUN conda install --yes --channel conda-forge/label/dev geopandas + +# update environemnt with common packages across worker and nb +COPY ../base_environment.yml /home/jovyan/base_environment.yml +RUN conda env update -f /home/jovyan/base_environment.yml + +# update environment with nb-specific packages COPY environment.yml /home/jovyan/environment.yml RUN conda env update -f /home/jovyan/environment.yml + +# create r environment COPY environment_r.yml /home/jovyan/environment_r.yml RUN conda env create -f /home/jovyan/environment_r.yml @@ -68,16 +78,6 @@ RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers USER $NB_USER -RUN pip install \ - rhg_compute_tools==0.1.8 \ - impactlab-tools==0.3.1 \ - climate-toolbox==0.1.5 \ - parameterize_jobs==0.1.1 \ - --no-cache-dir - -# update pip -RUN pip install --upgrade pip - WORKDIR $HOME COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json diff --git a/notebook/environment.yml b/notebook/environment.yml index 6f1d81b..971cf45 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -2,102 +2,16 @@ name: base channels: - conda-forge dependencies: - - beautifulsoup4=4.8.0 - - bokeh=1.3.4 - - bqplot=0.11.8 - - cartopy=0.17.0 - - click - - cloudpickle=1.2.2 - - cytoolz=0.10.0 - - dask-core=2.4.0 - - dask-glm=0.2.0 - - dask-ml=1.0.0 - - dask=2.4.0 - - datashader=0.7.0 - - distributed=2.4.0 - - esmf=7.1.0 - - esmpy=7.1.0 - - fastparquet=0.3.2 - - fiona=1.8.6 - - fusepy - - gcsfs=0.3.0 - - gdal=2.4.2 - - geoalchemy2=0.6.3 - - geopandas=0.5.1 - - geopy=1.20.0 - - geotiff=1.5.1 - - geoviews=1.6.3 - - git=2.23.0 - - gitpython=3.0.2 - - holoviews=1.12.5 - - icu=64.2 - - ipdb=0.12.2 - - ipython=7.8.0 - - ipyleaflet=0.11.2 - - ipywidgets=7.5.1 - - jedi=0.15.1 - - jupyter_client=5.3.3 - - jupyter=1.0.0 - - jupyterhub=1.0.0 - - jupyterlab=1.1.4 - - jupyter-server-proxy - - lz4=2.2.1 - - matplotlib=3.1.1 - - nb_conda_kernels=2.2 - - nbconvert=5.6.0 - - ncurses - - netcdf-fortran - - netcdf4=1.5.1.2 + - dask-labextension + - graphviz + - ipywidgets + - nbconvert - nodejs - nose - - notebook=6.0.1 - - numba=0.45.1 - - numcodecs=0.6.3 - - numpy - - pandas - - pip - - plotly - - pyshp=2.1.0 - - python-blosc=1.8.1 - - python-snappy=0.5.4 - - pyviz_comms=0.7.2 - - PyYAML - - rasterio=1.0.28 - - scikit-image=0.15.0 - - scikit-learn=0.21.3 - - scipy=1.3.1 - - seaborn=0.9.0 - - shapely=1.6.4 - - statsmodels=0.10.1 - - toolz=0.10.0 - - tornado=6.0.3 - - tqdm=4.36.1 - - urllib3=1.25.6 - - widgetsnbextension=3 - - xarray=0.13.0 - - zarr=2.3.2 - - zeromq=4.3.2 - - zict=1.0.0 - - pip: - - bumpversion - - coverage - - dask-labextension - - dask-kubernetes==0.9.2 - - flake8 - - google-api-core - - google-cloud - - google-cloud-container - - google-cloud-storage - - kubernetes - - mapbox - - py-noaa - - pytest - - pytest-cov - - pytest-runner - - sidecar - - Sphinx - - tox - - wget - - wheel - - xesmf - - xlrd + - jupyterhub + - jupyterlab + - pytest + - pytest-cov + - pytest-runner + - sphinx + - widgetsnbextension diff --git a/notebook/requirements.txt b/notebook/requirements.txt deleted file mode 100644 index eaaab72..0000000 --- a/notebook/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -statsd==3.2.1 -jupyterhub-dummyauthenticator==0.3.1 -jupyterhub-tmpauthenticator==0.4 -jupyterhub-ltiauthenticator==0.2 -nullauthenticator==1.0 -pymysql==0.7.11 -psycopg2==2.7.3.2 -pycurl==7.43.0 -mwoauth==0.3.2 -globus_sdk[jwt]==1.2.1 -oauthenticator==0.7.2 -cryptography==2.3 -https://github.com/jupyterhub/kubespawner/archive/86386e8.tar.gz diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml index 9e673f3..37f3c54 100644 --- a/notebook/worker-template.yml +++ b/notebook/worker-template.yml @@ -14,7 +14,7 @@ spec: env: - name: GCSFUSE_BUCKET value: rhg-data - image: rhodium/worker:d15fbe5fa4c4f78740cd3ab566007adf2f801947 + image: rhodium/worker:dev-update-packages name: dask-worker securityContext: capabilities: diff --git a/worker/Dockerfile b/worker/Dockerfile index 5efead3..4b36822 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -1,4 +1,27 @@ -FROM climateimpactlab/miniconda3-gfortran-netcdf:v1.0.2 +# using same container as jupyter/base-notebook:python-3.7.3 +ARG BASE_CONTAINER=ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c +FROM $BASE_CONTAINER + +USER root + +########### +# adapted from continuumio/miniconda3:4.6.14 +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV PATH /opt/conda/bin:$PATH + +RUN apt-get update --fix-missing && \ + apt-get install -y wget bzip2 ca-certificates curl git dumb-init && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + /opt/conda/bin/conda clean -tipsy && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc +########### # build conda @@ -7,8 +30,13 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda -COPY environment.yml /opt/conda/environment.yml -RUN conda env create -f /opt/conda/environment.yml +# update environemnt with common packages across worker and nb +COPY ../base_environment.yml /home/jovyan/base_environment.yml +RUN conda env create -f /home/jovyan/base_environment.yml -n worker + +# update environment with nb-specific packages +COPY environment.yml /home/jovyan/environment.yml +RUN conda env update -f /home/jovyan/environment.yml RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ @@ -40,13 +68,4 @@ RUN chmod +x /usr/bin/prepare.sh RUN mkdir /opt/app RUN mkdir /gcs -RUN /opt/conda/envs/worker/bin/pip install \ - rhg_compute_tools==0.1.0 \ - impactlab-tools==0.3.1 \ - climate-toolbox==0.1.4 \ - --no-cache-dir - -# update pip -RUN /opt/conda/envs/worker/bin/pip install --upgrade pip - ENTRYPOINT ["/usr/local/bin/dumb-init", "/usr/bin/prepare.sh"] diff --git a/worker/environment.yml b/worker/environment.yml index a1b9879..2b02ddf 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -2,94 +2,5 @@ name: worker channels: - conda-forge dependencies: - - beautifulsoup4=4.8.0 - - bokeh=1.3.4 - - bqplot=0.11.8 - - cartopy=0.17.0 - - click - - cloudpickle=1.2.2 - - cytoolz=0.10.0 - - dask-core=2.4.0 - - dask-glm=0.2.0 - - dask-ml=1.0.0 - - dask=2.4.0 - - datashader=0.7.0 - - distributed=2.4.0 - - esmf=7.1.0 - - esmpy=7.1.0 - - fastparquet=0.3.2 - - fiona=1.8.6 - - fusepy - - gcsfs=0.3.0 - - gdal=2.4.2 - - geoalchemy2=0.6.3 - - geopandas=0.5.1 - - geopy=1.20.0 - - geotiff=1.5.1 - - geoviews=1.6.3 - - git=2.23.0 - - gitpython=3.0.2 - - holoviews=1.12.5 - - icu=64.2 - - ipdb=0.12.2 - - ipython=7.8.0 - - ipyleaflet=0.11.2 - - ipywidgets=7.5.1 - - jedi=0.15.1 - - jupyter_client=5.3.3 - - jupyter=1.0.0 - - lz4=2.2.1 - - matplotlib=3.1.1 - - nb_conda_kernels=2.2 - - ncurses - - netcdf-fortran - - netcdf4=1.5.1.2 - - nose - - numba=0.45.1 - - numcodecs=0.6.3 - - numpy - - pandas - - pip - - plotly - - pyshp=2.1.0 - - python-blosc=1.8.1 - - python-snappy=0.5.4 - - pyviz_comms=0.7.2 - - PyYAML - - rasterio=1.0.28 - - scikit-image=0.15.0 - - scikit-learn=0.21.3 - - scipy=1.3.1 - - seaborn=0.9.0 - - shapely=1.6.4 - - statsmodels=0.10.1 - - toolz=0.10.0 - - tornado=6.0.3 - - tqdm=4.36.1 - - urllib3=1.25.6 - - xarray=0.13.0 - - zarr=2.3.2 - - zeromq=4.3.2 - - zict=1.0.0 - - pip: - - bumpversion - - coverage - - dask-kubernetes==0.9.2 - - flake8 - - google-api-core - - google-cloud - - google-cloud-container - - google-cloud-storage - - kubernetes - - mapbox - - py-noaa - - pytest - - pytest-cov - - pytest-runner - - sidecar - - Sphinx - - tox - - wget - - wheel - - xesmf - - xlrd + - jupyter + - jupyter_client From 08012840012a948294c58a17b3efa753ae82d5e3 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 4 Oct 2019 23:06:34 -0700 Subject: [PATCH 045/156] move base_environment in travis --- .travis.yml | 1 + base_environment.yml | 1 + notebook/Dockerfile | 3 ++- worker/Dockerfile | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 94daf86..8284420 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ install: - "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" - "rm notebook/worker-template.yml.bak" - "cat notebook/worker-template.yml | grep image:" +- "cp base_environment.yml $IMAGE_NAME/base_environment.yml" - "cd $IMAGE_NAME" - "docker pull rhodium/$IMAGE_NAME:dev" - "docker build --pull --cache-from rhodium/$IMAGE_NAME:dev -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." diff --git a/base_environment.yml b/base_environment.yml index cf8144e..1923d70 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -40,6 +40,7 @@ dependencies: - jedi - kubernetes - lz4=2.2.1 + - make - matplotlib - nb_conda_kernels - ncurses diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 4283328..9f34bea 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -6,12 +6,13 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes conda +RUN conda update --yes conda # install cythonized version of geopandas RUN conda install --yes --channel conda-forge/label/dev geopandas # update environemnt with common packages across worker and nb -COPY ../base_environment.yml /home/jovyan/base_environment.yml +COPY base_environment.yml /home/jovyan/base_environment.yml RUN conda env update -f /home/jovyan/base_environment.yml # update environment with nb-specific packages diff --git a/worker/Dockerfile b/worker/Dockerfile index 4b36822..48d240d 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update --fix-missing && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ /opt/conda/bin/conda clean -tipsy && \ @@ -31,7 +31,7 @@ RUN conda config --add channels conda-forge RUN conda update --yes conda # update environemnt with common packages across worker and nb -COPY ../base_environment.yml /home/jovyan/base_environment.yml +COPY base_environment.yml /home/jovyan/base_environment.yml RUN conda env create -f /home/jovyan/base_environment.yml -n worker # update environment with nb-specific packages From f2f92193080ef49c032fd9430ca62cc49e7a02c2 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 4 Oct 2019 23:24:16 -0700 Subject: [PATCH 046/156] fix bad package names --- base_environment.yml | 3 ++- notebook/Dockerfile | 3 +-- worker/Dockerfile | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 1923d70..ac1756f 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -53,7 +53,7 @@ dependencies: - plotly - pyshp - python-blosc - - python-snapp + - python-snappy - pyviz_comms - PyYAML - rasterio @@ -80,6 +80,7 @@ dependencies: - pip: - mapbox - py-noaa + - sidecar - rhg_compute_tools - impactlab-tools - climate-toolbox diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 9f34bea..5017271 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -5,8 +5,7 @@ FROM jupyter/base-notebook:python-3.7.3 USER $NB_USER RUN conda config --add channels conda-forge -RUN conda update --yes conda -RUN conda update --yes conda +RUN conda update --yes -n base conda # install cythonized version of geopandas RUN conda install --yes --channel conda-forge/label/dev geopandas diff --git a/worker/Dockerfile b/worker/Dockerfile index 48d240d..4500a17 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update --fix-missing && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh -O ~/miniconda.sh && \ +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ /opt/conda/bin/conda clean -tipsy && \ @@ -28,7 +28,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.12-Linux-x86 USER $NB_USER RUN conda config --add channels conda-forge -RUN conda update --yes conda +RUN conda update --yes -n base conda # update environemnt with common packages across worker and nb COPY base_environment.yml /home/jovyan/base_environment.yml From 2e00d6931a0f68d0af36bea503abfa6e410e64ad Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 4 Oct 2019 23:38:05 -0700 Subject: [PATCH 047/156] fix sidecar --- base_environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index ac1756f..10e078b 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -62,7 +62,6 @@ dependencies: - scipy - seaborn - shapely - - sidecar - statsmodels - toolz - tornado From 1018b4e1cf453333234f9373ce92f1a422db2e2b Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 5 Oct 2019 00:38:02 -0700 Subject: [PATCH 048/156] apt-get-installs.sh --- .travis.yml | 1 + apt-get-installs.sh | 25 +++++++++++++++++++++++++ base_environment.yml | 5 +---- notebook/Dockerfile | 21 ++++++--------------- notebook/environment.yml | 2 ++ worker/Dockerfile | 29 +++++++---------------------- 6 files changed, 42 insertions(+), 41 deletions(-) create mode 100644 apt-get-installs.sh diff --git a/.travis.yml b/.travis.yml index 8284420..a48f895 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ install: - "rm notebook/worker-template.yml.bak" - "cat notebook/worker-template.yml | grep image:" - "cp base_environment.yml $IMAGE_NAME/base_environment.yml" +- "cp apt-get-installs.sh $IMAGE_NAME/apt-get-installs.sh && chmod +x $IMAGE_NAME/apt-get-installs.sh" - "cd $IMAGE_NAME" - "docker pull rhodium/$IMAGE_NAME:dev" - "docker build --pull --cache-from rhodium/$IMAGE_NAME:dev -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." diff --git a/apt-get-installs.sh b/apt-get-installs.sh new file mode 100644 index 0000000..9948435 --- /dev/null +++ b/apt-get-installs.sh @@ -0,0 +1,25 @@ +apt-get update --fix-missing --no-install-recommends + +apt-get install -yq --no-install-recommends apt-utils \ + wget bzip2 ca-certificates curl git gnupg2 apt-transport-https + +# install google cloud sdk +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ + https://packages.cloud.google.com/apt cloud-sdk main" | \ + sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ + sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - +apt-get update -y +apt-get install google-cloud-sdk kubectl -y + +# install gcsFUSE +export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s` +echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +apt-get update -y +apt-get install gcsfuse -y +alias googlefuse=/usr/bin/gcsfuse + +apt-get upgrade -yq --no-install-recommends + +apt-get clean diff --git a/base_environment.yml b/base_environment.yml index 10e078b..553924e 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -9,7 +9,6 @@ dependencies: - cartopy - click - cloudpickle - - coverage - cytoolz - dask - dask-glm @@ -21,7 +20,6 @@ dependencies: - esmpy - fastparquet - fiona - - flake8 - fusepy - gcsfs - gdal @@ -39,7 +37,7 @@ dependencies: - ipyleaflet - jedi - kubernetes - - lz4=2.2.1 + - lz4 - make - matplotlib - nb_conda_kernels @@ -68,7 +66,6 @@ dependencies: - tox - tqdm - urllib3 - - wget - wheel - xarray - xesmf diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 5017271..e572c2d 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -2,6 +2,11 @@ FROM jupyter/base-notebook:python-3.7.3 # build conda +USER root +RUN bash apt-get-installs.sh +RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy +RUN chmod +x /usr/bin/cloud_sql_proxy + USER $NB_USER RUN conda config --add channels conda-forge @@ -50,24 +55,10 @@ COPY run_sql_proxy.py /pre-home COPY config.yaml /pre-home COPY worker-template.yml /pre-home -RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ - echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get update -y && apt-get install google-cloud-sdk kubectl -y - -RUN export GCSFUSE_REPO=gcsfuse-xenial \ - && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ - && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ - && apt-get update \ - && apt-get install gcsfuse -y \ - && alias googlefuse=/usr/bin/gcsfuse - USER root -RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy -RUN chmod +x /usr/bin/cloud_sql_proxy # clean up -RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN rm -rf /var/lib/apt/lists/* RUN conda clean -tipsy RUN mkdir /gcs && chown -R $NB_USER /gcs diff --git a/notebook/environment.yml b/notebook/environment.yml index 971cf45..1b85d3f 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -2,7 +2,9 @@ name: base channels: - conda-forge dependencies: + - coverage - dask-labextension + - flake8 - graphviz - ipywidgets - nbconvert diff --git a/worker/Dockerfile b/worker/Dockerfile index 4500a17..a9cd4f9 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -3,17 +3,15 @@ ARG BASE_CONTAINER=ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd FROM $BASE_CONTAINER USER root +RUN bash apt-get-installs.sh +RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy +RUN chmod +x /usr/bin/cloud_sql_proxy ########### # adapted from continuumio/miniconda3:4.6.14 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PATH /opt/conda/bin:$PATH -RUN apt-get update --fix-missing && \ - apt-get install -y wget bzip2 ca-certificates curl git dumb-init && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ @@ -30,6 +28,9 @@ USER $NB_USER RUN conda config --add channels conda-forge RUN conda update --yes -n base conda +# install cythonized version of geopandas +RUN conda install --yes -n worker --channel conda-forge/label/dev geopandas + # update environemnt with common packages across worker and nb COPY base_environment.yml /home/jovyan/base_environment.yml RUN conda env create -f /home/jovyan/base_environment.yml -n worker @@ -38,29 +39,13 @@ RUN conda env create -f /home/jovyan/base_environment.yml -n worker COPY environment.yml /home/jovyan/environment.yml RUN conda env update -f /home/jovyan/environment.yml -RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ - echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get update -y && apt-get install google-cloud-sdk kubectl -y - -RUN export GCSFUSE_REPO=gcsfuse-xenial \ - && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ - && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ - && apt-get update \ - && apt-get install gcsfuse -y \ - && alias googlefuse=/usr/bin/gcsfuse - # clean up -RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN rm -rf /var/lib/apt/lists/* RUN conda clean -tipsy ENV OMP_NUM_THREADS=1 ENV DASK_TICK_MAXIMUM_DELAY=5s -USER root -RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy -RUN chmod +x /usr/bin/cloud_sql_proxy - USER root COPY prepare.sh /usr/bin/prepare.sh COPY add_service_creds.py /usr/bin/add_service_creds.py From 7e957fc1676b19971b1462974a8965aba84be1e6 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 5 Oct 2019 00:51:54 -0700 Subject: [PATCH 049/156] fix run apt-get --- notebook/Dockerfile | 2 +- worker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index e572c2d..1fe883c 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -3,7 +3,7 @@ FROM jupyter/base-notebook:python-3.7.3 # build conda USER root -RUN bash apt-get-installs.sh +RUN apt-get-installs.sh RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy RUN chmod +x /usr/bin/cloud_sql_proxy diff --git a/worker/Dockerfile b/worker/Dockerfile index a9cd4f9..e095ac6 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -3,7 +3,7 @@ ARG BASE_CONTAINER=ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd FROM $BASE_CONTAINER USER root -RUN bash apt-get-installs.sh +RUN apt-get-installs.sh RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy RUN chmod +x /usr/bin/cloud_sql_proxy From d40e4288390f6c638b4c075a66d001590575f435 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 5 Oct 2019 08:08:29 -0700 Subject: [PATCH 050/156] cleaning up dockerfile --- apt-get-installs.sh => common.sh | 16 ++++++++- notebook/Dockerfile | 61 ++++++++++++++------------------ worker/Dockerfile | 37 ++++++++++--------- 3 files changed, 59 insertions(+), 55 deletions(-) rename apt-get-installs.sh => common.sh (70%) diff --git a/apt-get-installs.sh b/common.sh similarity index 70% rename from apt-get-installs.sh rename to common.sh index 9948435..2fd65ab 100644 --- a/apt-get-installs.sh +++ b/common.sh @@ -1,4 +1,5 @@ -apt-get update --fix-missing --no-install-recommends + #!/bin/sh + apt-get update --fix-missing --no-install-recommends apt-get install -yq --no-install-recommends apt-utils \ wget bzip2 ca-certificates curl git gnupg2 apt-transport-https @@ -23,3 +24,16 @@ alias googlefuse=/usr/bin/gcsfuse apt-get upgrade -yq --no-install-recommends apt-get clean + +# get cloud sql proxy +wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy +chmod +x /usr/bin/cloud_sql_proxy + +# set up conda +conda config --add channels conda-forge +conda update --yes -n base conda + +# filepath curating +chmod +x /usr/bin/prepare.sh +mkdir /gcs +mkdir /opt/app diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 1fe883c..63ed363 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,30 +1,42 @@ FROM jupyter/base-notebook:python-3.7.3 -# build conda +## Add NB_USER to sudo +RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook +RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers +USER $NB_USER -USER root -RUN apt-get-installs.sh -RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy -RUN chmod +x /usr/bin/cloud_sql_proxy -USER $NB_USER +## filepath curation +COPY common.sh /home/jovyan/common.sh +COPY prepare.sh /usr/bin/prepare.sh +COPY base_environment.yml /home/jovyan/base_environment.yml +COPY environment.yml /home/jovyan/environment.yml +COPY environment_r.yml /home/jovyan/environment_r.yml +COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json -RUN conda config --add channels conda-forge -RUN conda update --yes -n base conda +RUN mkdir /pre-home && chown -R $NB_USER /pre-home +COPY worker-template.yml /pre-home +COPY add_service_creds.py /pre-home +COPY run_sql_proxy.py /pre-home +COPY config.yaml /pre-home +COPY worker-template.yml /pre-home + +## perform a bunch of common actions +RUN /home/jovyan/common.sh + + +## set up python env # install cythonized version of geopandas RUN conda install --yes --channel conda-forge/label/dev geopandas # update environemnt with common packages across worker and nb -COPY base_environment.yml /home/jovyan/base_environment.yml RUN conda env update -f /home/jovyan/base_environment.yml # update environment with nb-specific packages -COPY environment.yml /home/jovyan/environment.yml RUN conda env update -f /home/jovyan/environment.yml # create r environment -COPY environment_r.yml /home/jovyan/environment_r.yml RUN conda env create -f /home/jovyan/environment_r.yml RUN jupyter labextension update --all && \ @@ -44,34 +56,13 @@ RUN jupyter serverextension enable --py jupyterlab --user RUN jupyter serverextension enable --sys-prefix jupyter_server_proxy RUN jupyter serverextension enable --py --sys-prefix dask_labextension -USER root -COPY prepare.sh /usr/bin/prepare.sh -RUN chmod +x /usr/bin/prepare.sh -RUN mkdir /pre-home && chown -R $NB_USER /pre-home -COPY worker-template.yml /pre-home -COPY add_service_creds.py /pre-home -COPY run_sql_proxy.py /pre-home - -COPY config.yaml /pre-home -COPY worker-template.yml /pre-home -USER root - -# clean up +## clean up RUN rm -rf /var/lib/apt/lists/* RUN conda clean -tipsy +RUN chown -R $NB_USER /gcs -RUN mkdir /gcs && chown -R $NB_USER /gcs -RUN mkdir /opt/app - -# Add NB_USER to sudo -RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook -RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers -USER $NB_USER - +## prepare container WORKDIR $HOME - -COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json - ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] CMD ["start.sh jupyter lab"] diff --git a/worker/Dockerfile b/worker/Dockerfile index e095ac6..b244bad 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -1,11 +1,20 @@ -# using same container as jupyter/base-notebook:python-3.7.3 +## using same container as jupyter/base-notebook:python-3.7.3 ARG BASE_CONTAINER=ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c FROM $BASE_CONTAINER + +## filepath curation USER root -RUN apt-get-installs.sh -RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy -RUN chmod +x /usr/bin/cloud_sql_proxy +COPY common.sh /home/jovyan/common.sh +COPY prepare.sh /usr/bin/prepare.sh +COPY add_service_creds.py /usr/bin/add_service_creds.py +COPY base_environment.yml /home/jovyan/base_environment.yml +COPY environment.yml /home/jovyan/environment.yml + + +## perform a bunch of common actions +RUN /home/jovyan/common.sh + ########### # adapted from continuumio/miniconda3:4.6.14 @@ -21,36 +30,26 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86 echo "conda activate base" >> ~/.bashrc ########### -# build conda +## set up python env USER $NB_USER -RUN conda config --add channels conda-forge -RUN conda update --yes -n base conda - # install cythonized version of geopandas RUN conda install --yes -n worker --channel conda-forge/label/dev geopandas # update environemnt with common packages across worker and nb -COPY base_environment.yml /home/jovyan/base_environment.yml RUN conda env create -f /home/jovyan/base_environment.yml -n worker # update environment with nb-specific packages -COPY environment.yml /home/jovyan/environment.yml RUN conda env update -f /home/jovyan/environment.yml -# clean up + +## clean up RUN rm -rf /var/lib/apt/lists/* RUN conda clean -tipsy + +## prepare container ENV OMP_NUM_THREADS=1 ENV DASK_TICK_MAXIMUM_DELAY=5s - -USER root -COPY prepare.sh /usr/bin/prepare.sh -COPY add_service_creds.py /usr/bin/add_service_creds.py -RUN chmod +x /usr/bin/prepare.sh -RUN mkdir /opt/app -RUN mkdir /gcs - ENTRYPOINT ["/usr/local/bin/dumb-init", "/usr/bin/prepare.sh"] From eca56450f0942d0e2fe9d76582724346a8c24d7e Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 5 Oct 2019 08:16:09 -0700 Subject: [PATCH 051/156] fix common file in travis --- .travis.yml | 2 +- notebook/Dockerfile | 5 +++-- worker/Dockerfile | 5 ++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a48f895..39d4922 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: - "rm notebook/worker-template.yml.bak" - "cat notebook/worker-template.yml | grep image:" - "cp base_environment.yml $IMAGE_NAME/base_environment.yml" -- "cp apt-get-installs.sh $IMAGE_NAME/apt-get-installs.sh && chmod +x $IMAGE_NAME/apt-get-installs.sh" +- "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh" - "cd $IMAGE_NAME" - "docker pull rhodium/$IMAGE_NAME:dev" - "docker build --pull --cache-from rhodium/$IMAGE_NAME:dev -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 63ed363..fb884b5 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -24,6 +24,7 @@ COPY worker-template.yml /pre-home ## perform a bunch of common actions RUN /home/jovyan/common.sh +RUN chown -R $NB_USER /gcs ## set up python env @@ -59,8 +60,8 @@ RUN jupyter serverextension enable --py --sys-prefix dask_labextension ## clean up RUN rm -rf /var/lib/apt/lists/* -RUN conda clean -tipsy -RUN chown -R $NB_USER /gcs +RUN conda clean --all -y -f + ## prepare container WORKDIR $HOME diff --git a/worker/Dockerfile b/worker/Dockerfile index b244bad..ef4af19 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -17,14 +17,13 @@ RUN /home/jovyan/common.sh ########### -# adapted from continuumio/miniconda3:4.6.14 +# adapted from continuumio/miniconda3 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PATH /opt/conda/bin:$PATH RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ - /opt/conda/bin/conda clean -tipsy && \ ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ echo "conda activate base" >> ~/.bashrc @@ -46,7 +45,7 @@ RUN conda env update -f /home/jovyan/environment.yml ## clean up RUN rm -rf /var/lib/apt/lists/* -RUN conda clean -tipsy +RUN conda clean --all -y -f ## prepare container From 609305e0480addeef6341d4abf2c66748edf1bcc Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 5 Oct 2019 08:38:41 -0700 Subject: [PATCH 052/156] fix conda install order --- notebook/Dockerfile | 6 ++++-- worker/Dockerfile | 10 +++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index fb884b5..c997301 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,6 +1,7 @@ FROM jupyter/base-notebook:python-3.7.3 ## Add NB_USER to sudo +USER root RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers USER $NB_USER @@ -28,12 +29,13 @@ RUN chown -R $NB_USER /gcs ## set up python env -# install cythonized version of geopandas -RUN conda install --yes --channel conda-forge/label/dev geopandas # update environemnt with common packages across worker and nb RUN conda env update -f /home/jovyan/base_environment.yml +# update with cythonized geopandas +RUN conda install -n worker --channel conda-forge/label/dev geopandas + # update environment with nb-specific packages RUN conda env update -f /home/jovyan/environment.yml diff --git a/worker/Dockerfile b/worker/Dockerfile index ef4af19..22e86fe 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -33,13 +33,13 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86 ## set up python env USER $NB_USER -# install cythonized version of geopandas -RUN conda install --yes -n worker --channel conda-forge/label/dev geopandas - -# update environemnt with common packages across worker and nb +# create environemnt with common packages across worker and nb RUN conda env create -f /home/jovyan/base_environment.yml -n worker -# update environment with nb-specific packages +# update with cythonized geopandas +RUN conda install -n worker --channel conda-forge/label/dev geopandas + +# update environment with worker-specific packages RUN conda env update -f /home/jovyan/environment.yml From b9e29475f7bc682d266727d139942a54174420b5 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 5 Oct 2019 09:01:09 -0700 Subject: [PATCH 053/156] user permission wrangling --- common.sh | 8 ++++---- notebook/Dockerfile | 5 +++-- worker/Dockerfile | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/common.sh b/common.sh index 2fd65ab..de0ed7d 100644 --- a/common.sh +++ b/common.sh @@ -1,5 +1,5 @@ - #!/bin/sh - apt-get update --fix-missing --no-install-recommends +#!/bin/sh +apt-get update --fix-missing --no-install-recommends apt-get install -yq --no-install-recommends apt-utils \ wget bzip2 ca-certificates curl git gnupg2 apt-transport-https @@ -7,9 +7,9 @@ apt-get install -yq --no-install-recommends apt-utils \ # install google cloud sdk echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ https://packages.cloud.google.com/apt cloud-sdk main" | \ - sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + tee -a /etc/apt/sources.list.d/google-cloud-sdk.list curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ - sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - apt-get update -y apt-get install google-cloud-sdk kubectl -y diff --git a/notebook/Dockerfile b/notebook/Dockerfile index c997301..106a6f9 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -4,7 +4,6 @@ FROM jupyter/base-notebook:python-3.7.3 USER root RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers -USER $NB_USER ## filepath curation @@ -24,11 +23,12 @@ COPY worker-template.yml /pre-home ## perform a bunch of common actions -RUN /home/jovyan/common.sh +RUN sudo bash /home/jovyan/common.sh RUN chown -R $NB_USER /gcs ## set up python env +USER $NB_USER # update environemnt with common packages across worker and nb RUN conda env update -f /home/jovyan/base_environment.yml @@ -61,6 +61,7 @@ RUN jupyter serverextension enable --py --sys-prefix dask_labextension ## clean up +USER root RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f diff --git a/worker/Dockerfile b/worker/Dockerfile index 22e86fe..e1d7b96 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -13,7 +13,7 @@ COPY environment.yml /home/jovyan/environment.yml ## perform a bunch of common actions -RUN /home/jovyan/common.sh +RUN sudo bash /home/jovyan/common.sh ########### @@ -44,6 +44,7 @@ RUN conda env update -f /home/jovyan/environment.yml ## clean up +USER root RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f From b62d1ab2e1f45ba898df36f38031198bff053873 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 5 Oct 2019 09:22:18 -0700 Subject: [PATCH 054/156] fix env name and sudo --- common.sh | 2 +- notebook/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common.sh b/common.sh index de0ed7d..32ead51 100644 --- a/common.sh +++ b/common.sh @@ -2,7 +2,7 @@ apt-get update --fix-missing --no-install-recommends apt-get install -yq --no-install-recommends apt-utils \ - wget bzip2 ca-certificates curl git gnupg2 apt-transport-https + wget bzip2 ca-certificates curl git gnupg2 apt-transport-https sudo # install google cloud sdk echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 106a6f9..adb78ca 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -34,7 +34,7 @@ USER $NB_USER RUN conda env update -f /home/jovyan/base_environment.yml # update with cythonized geopandas -RUN conda install -n worker --channel conda-forge/label/dev geopandas +RUN conda install --channel conda-forge/label/dev geopandas # update environment with nb-specific packages RUN conda env update -f /home/jovyan/environment.yml From 3216608715f3c9d9c123c8de7873e70349182d9e Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 5 Oct 2019 23:20:39 -0700 Subject: [PATCH 055/156] Update Dockerfile --- notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index adb78ca..2533641 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -4,7 +4,7 @@ FROM jupyter/base-notebook:python-3.7.3 USER root RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers - +USER $NB_USER ## filepath curation COPY common.sh /home/jovyan/common.sh @@ -28,7 +28,6 @@ RUN chown -R $NB_USER /gcs ## set up python env -USER $NB_USER # update environemnt with common packages across worker and nb RUN conda env update -f /home/jovyan/base_environment.yml @@ -68,5 +67,6 @@ RUN conda clean --all -y -f ## prepare container WORKDIR $HOME +USER $NB_USER ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] CMD ["start.sh jupyter lab"] From 26d67b4c375f9a9a980b39be0f71b3c5db9b9240 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 00:11:51 -0700 Subject: [PATCH 056/156] permissions --- common.sh | 2 +- notebook/Dockerfile | 8 ++++---- worker/Dockerfile | 11 +++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/common.sh b/common.sh index 32ead51..de0ed7d 100644 --- a/common.sh +++ b/common.sh @@ -2,7 +2,7 @@ apt-get update --fix-missing --no-install-recommends apt-get install -yq --no-install-recommends apt-utils \ - wget bzip2 ca-certificates curl git gnupg2 apt-transport-https sudo + wget bzip2 ca-certificates curl git gnupg2 apt-transport-https # install google cloud sdk echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 2533641..1040bee 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -4,15 +4,11 @@ FROM jupyter/base-notebook:python-3.7.3 USER root RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers -USER $NB_USER ## filepath curation COPY common.sh /home/jovyan/common.sh COPY prepare.sh /usr/bin/prepare.sh COPY base_environment.yml /home/jovyan/base_environment.yml -COPY environment.yml /home/jovyan/environment.yml -COPY environment_r.yml /home/jovyan/environment_r.yml -COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json RUN mkdir /pre-home && chown -R $NB_USER /pre-home COPY worker-template.yml /pre-home @@ -28,6 +24,10 @@ RUN chown -R $NB_USER /gcs ## set up python env +USER $NB_USER +COPY environment.yml /home/jovyan/environment.yml +COPY environment_r.yml /home/jovyan/environment_r.yml +COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json # update environemnt with common packages across worker and nb RUN conda env update -f /home/jovyan/base_environment.yml diff --git a/worker/Dockerfile b/worker/Dockerfile index e1d7b96..eb846f9 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -2,15 +2,15 @@ ARG BASE_CONTAINER=ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c FROM $BASE_CONTAINER +## get sudo +RUN apt-get update --fix-missing --no-install-recommends && \ + apt-get install -yq --no-install-recommends sudo + ## filepath curation USER root COPY common.sh /home/jovyan/common.sh COPY prepare.sh /usr/bin/prepare.sh -COPY add_service_creds.py /usr/bin/add_service_creds.py -COPY base_environment.yml /home/jovyan/base_environment.yml -COPY environment.yml /home/jovyan/environment.yml - ## perform a bunch of common actions RUN sudo bash /home/jovyan/common.sh @@ -32,6 +32,9 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86 ## set up python env USER $NB_USER +COPY add_service_creds.py /usr/bin/add_service_creds.py +COPY base_environment.yml /home/jovyan/base_environment.yml +COPY environment.yml /home/jovyan/environment.yml # create environemnt with common packages across worker and nb RUN conda env create -f /home/jovyan/base_environment.yml -n worker From bee583996464ad7cce8e6aed05efd49a08ccdda2 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 00:30:32 -0700 Subject: [PATCH 057/156] another permissions try --- common.sh | 4 ---- notebook/Dockerfile | 5 +---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/common.sh b/common.sh index de0ed7d..10c9808 100644 --- a/common.sh +++ b/common.sh @@ -29,10 +29,6 @@ apt-get clean wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy chmod +x /usr/bin/cloud_sql_proxy -# set up conda -conda config --add channels conda-forge -conda update --yes -n base conda - # filepath curating chmod +x /usr/bin/prepare.sh mkdir /gcs diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 1040bee..7054d37 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,9 +1,9 @@ FROM jupyter/base-notebook:python-3.7.3 ## Add NB_USER to sudo -USER root RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers +USER $NB_USER ## filepath curation COPY common.sh /home/jovyan/common.sh @@ -24,7 +24,6 @@ RUN chown -R $NB_USER /gcs ## set up python env -USER $NB_USER COPY environment.yml /home/jovyan/environment.yml COPY environment_r.yml /home/jovyan/environment_r.yml COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json @@ -60,13 +59,11 @@ RUN jupyter serverextension enable --py --sys-prefix dask_labextension ## clean up -USER root RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f ## prepare container WORKDIR $HOME -USER $NB_USER ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] CMD ["start.sh jupyter lab"] From 4f1d2565cb8835ff5a754acf3916348c1698e6c9 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 09:04:07 -0700 Subject: [PATCH 058/156] try cloud-sdk from conda --- base_environment.yml | 1 + common.sh | 35 +++++++++++++++++------------------ notebook/Dockerfile | 9 +++++++-- worker/Dockerfile | 7 +++++-- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 553924e..6d3d443 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -29,6 +29,7 @@ dependencies: - geoviews - git - gitpython + - google-cloud-sdk - google-cloud-container - google-cloud-storage - holoviews diff --git a/common.sh b/common.sh index 10c9808..4338233 100644 --- a/common.sh +++ b/common.sh @@ -1,30 +1,29 @@ #!/bin/sh -apt-get update --fix-missing --no-install-recommends -apt-get install -yq --no-install-recommends apt-utils \ - wget bzip2 ca-certificates curl git gnupg2 apt-transport-https - -# install google cloud sdk -echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ - https://packages.cloud.google.com/apt cloud-sdk main" | \ - tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ - apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - -apt-get update -y -apt-get install google-cloud-sdk kubectl -y - -# install gcsFUSE +# add gcsfuse dist. URL to apt as package source & import public key export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s` echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - -apt-get update -y -apt-get install gcsfuse -y -alias googlefuse=/usr/bin/gcsfuse +# install apt-get packages +apt-get update --no-install-recommends -y +apt-get install -yq --no-install-recommends apt-utils \ + wget bzip2 ca-certificates curl git gnupg2 gcsfuse apt-get upgrade -yq --no-install-recommends - apt-get clean +# set up aliases +alias googlefuse=/usr/bin/gcsfuse + +# install google cloud sdk +# echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ +# https://packages.cloud.google.com/apt cloud-sdk main" | \ +# tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +# curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ +# apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - +# apt-get update -y +# apt-get install google-cloud-sdk kubectl -y + # get cloud sql proxy wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy chmod +x /usr/bin/cloud_sql_proxy diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 7054d37..b8ad872 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,9 +1,10 @@ FROM jupyter/base-notebook:python-3.7.3 ## Add NB_USER to sudo +USER root RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers -USER $NB_USER + ## filepath curation COPY common.sh /home/jovyan/common.sh @@ -19,7 +20,7 @@ COPY worker-template.yml /pre-home ## perform a bunch of common actions -RUN sudo bash /home/jovyan/common.sh +RUN bash /home/jovyan/common.sh RUN chown -R $NB_USER /gcs @@ -58,6 +59,10 @@ RUN jupyter serverextension enable --sys-prefix jupyter_server_proxy RUN jupyter serverextension enable --py --sys-prefix dask_labextension +## install kubectl +gcloud components install kubectl + + ## clean up RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f diff --git a/worker/Dockerfile b/worker/Dockerfile index eb846f9..93c39b9 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -8,12 +8,11 @@ RUN apt-get update --fix-missing --no-install-recommends && \ ## filepath curation -USER root COPY common.sh /home/jovyan/common.sh COPY prepare.sh /usr/bin/prepare.sh ## perform a bunch of common actions -RUN sudo bash /home/jovyan/common.sh +RUN bash /home/jovyan/common.sh ########### @@ -46,6 +45,10 @@ RUN conda install -n worker --channel conda-forge/label/dev geopandas RUN conda env update -f /home/jovyan/environment.yml +## install kubectl +gcloud components install kubectl + + ## clean up USER root RUN rm -rf /var/lib/apt/lists/* From aaa4acc0c4f578c0bcefbb97151d32a49865daf1 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 09:12:50 -0700 Subject: [PATCH 059/156] removing docker cache temporarily --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 39d4922..075a157 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,7 @@ install: - "cp base_environment.yml $IMAGE_NAME/base_environment.yml" - "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh" - "cd $IMAGE_NAME" -- "docker pull rhodium/$IMAGE_NAME:dev" -- "docker build --pull --cache-from rhodium/$IMAGE_NAME:dev -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." +- "docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." script: - docker images rhodium/$IMAGE_NAME:$TRAVIS_COMMIT From 12c98c9bccabaa3b8395c4156d63a6aced4780a5 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 09:15:21 -0700 Subject: [PATCH 060/156] fix gcloud command --- notebook/Dockerfile | 2 +- worker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index b8ad872..2904e70 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -60,7 +60,7 @@ RUN jupyter serverextension enable --py --sys-prefix dask_labextension ## install kubectl -gcloud components install kubectl +RUN gcloud components install kubectl ## clean up diff --git a/worker/Dockerfile b/worker/Dockerfile index 93c39b9..cde1be2 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -46,7 +46,7 @@ RUN conda env update -f /home/jovyan/environment.yml ## install kubectl -gcloud components install kubectl +RUN gcloud components install kubectl ## clean up From 29a83d6dfb5d646add1740e032bb55a24e9c4735 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 09:21:54 -0700 Subject: [PATCH 061/156] rearrange apt-get commands --- common.sh | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/common.sh b/common.sh index 4338233..03d020e 100644 --- a/common.sh +++ b/common.sh @@ -1,20 +1,25 @@ #!/bin/sh -# add gcsfuse dist. URL to apt as package source & import public key +# install apt-get packages +apt-get update --no-install-recommends -y +apt-get install -yq --no-install-recommends \ + apt-utils \ + wget \ + bzip2 \ + ca-certificates \ + curl \ + gnupg2 + +# install gcsfuse +# (need curl to be installed earlier) export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s` echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +apt-get install -yq --no-install-recommends gcsfuse +alias googlefuse=/usr/bin/gcsfuse -# install apt-get packages -apt-get update --no-install-recommends -y -apt-get install -yq --no-install-recommends apt-utils \ - wget bzip2 ca-certificates curl git gnupg2 gcsfuse -apt-get upgrade -yq --no-install-recommends apt-get clean -# set up aliases -alias googlefuse=/usr/bin/gcsfuse - # install google cloud sdk # echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ # https://packages.cloud.google.com/apt cloud-sdk main" | \ From a3a8172d2bfbc25751f7bbde4910a90a0a7f4b44 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 10:00:44 -0700 Subject: [PATCH 062/156] new google cloud install --- base_environment.yml | 1 - common.sh | 30 ++++++++++++++++-------------- notebook/Dockerfile | 1 + worker/Dockerfile | 18 +++++++++--------- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 6d3d443..553924e 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -29,7 +29,6 @@ dependencies: - geoviews - git - gitpython - - google-cloud-sdk - google-cloud-container - google-cloud-storage - holoviews diff --git a/common.sh b/common.sh index 03d020e..506a3eb 100644 --- a/common.sh +++ b/common.sh @@ -4,35 +4,37 @@ apt-get update --no-install-recommends -y apt-get install -yq --no-install-recommends \ apt-utils \ - wget \ bzip2 \ ca-certificates \ curl \ - gnupg2 + lsb-release \ + gnupg2 \ + sudo -# install gcsfuse +# install gcsfuse, google cloud sdk, kubectl # (need curl to be installed earlier) export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s` -echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list +echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | \ + tee /etc/apt/sources.list.d/gcsfuse.list +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ + https://packages.cloud.google.com/apt cloud-sdk main" | \ + tee -a /etc/apt/sources.list.d/google-cloud-sdk.list curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - -apt-get install -yq --no-install-recommends gcsfuse +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ + apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - +apt-get install -yq --no-install-recommends gcsfuse google-cloud-sdk kubectl alias googlefuse=/usr/bin/gcsfuse apt-get clean -# install google cloud sdk -# echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ -# https://packages.cloud.google.com/apt cloud-sdk main" | \ -# tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -# curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ -# apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - -# apt-get update -y -# apt-get install google-cloud-sdk kubectl -y - # get cloud sql proxy wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy chmod +x /usr/bin/cloud_sql_proxy +# conda updates +conda update -n base conda +conda config --set channel_priority strict + # filepath curating chmod +x /usr/bin/prepare.sh mkdir /gcs diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 2904e70..12e61c6 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,4 +1,5 @@ FROM jupyter/base-notebook:python-3.7.3 +ARG DEBIAN_FRONTEND=noninteractive ## Add NB_USER to sudo USER root diff --git a/worker/Dockerfile b/worker/Dockerfile index cde1be2..a0ed617 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -1,21 +1,17 @@ ## using same container as jupyter/base-notebook:python-3.7.3 -ARG BASE_CONTAINER=ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c -FROM $BASE_CONTAINER - -## get sudo -RUN apt-get update --fix-missing --no-install-recommends && \ - apt-get install -yq --no-install-recommends sudo +FROM ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c +ARG DEBIAN_FRONTEND=noninteractive ## filepath curation COPY common.sh /home/jovyan/common.sh COPY prepare.sh /usr/bin/prepare.sh -## perform a bunch of common actions -RUN bash /home/jovyan/common.sh - ########### +apt-get update --no-install-recommends -y +apt-get install -yq --no-install-recommends wget + # adapted from continuumio/miniconda3 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PATH /opt/conda/bin:$PATH @@ -29,6 +25,10 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86 ########### +## perform a bunch of common actions +RUN bash /home/jovyan/common.sh + + ## set up python env USER $NB_USER COPY add_service_creds.py /usr/bin/add_service_creds.py From fd150423b5ffe39548c1f33ba977b93280240360 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 10:22:48 -0700 Subject: [PATCH 063/156] pin python for geopandas --- base_environment.yml | 1 + common.sh | 4 +++- worker/Dockerfile | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 553924e..4143b8c 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -50,6 +50,7 @@ dependencies: - pip - plotly - pyshp + - python=3.6 - python-blosc - python-snappy - pyviz_comms diff --git a/common.sh b/common.sh index 506a3eb..0b0d2d2 100644 --- a/common.sh +++ b/common.sh @@ -1,7 +1,7 @@ #!/bin/sh # install apt-get packages -apt-get update --no-install-recommends -y +apt-get update -y apt-get install -yq --no-install-recommends \ apt-utils \ bzip2 \ @@ -22,6 +22,7 @@ echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - +apt-get update -y apt-get install -yq --no-install-recommends gcsfuse google-cloud-sdk kubectl alias googlefuse=/usr/bin/gcsfuse @@ -34,6 +35,7 @@ chmod +x /usr/bin/cloud_sql_proxy # conda updates conda update -n base conda conda config --set channel_priority strict +cat ~/.condarc # filepath curating chmod +x /usr/bin/prepare.sh diff --git a/worker/Dockerfile b/worker/Dockerfile index a0ed617..5a987a8 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -9,8 +9,8 @@ COPY prepare.sh /usr/bin/prepare.sh ########### -apt-get update --no-install-recommends -y -apt-get install -yq --no-install-recommends wget +RUN apt-get update --no-install-recommends -y +RUN apt-get install -yq --no-install-recommends wget # adapted from continuumio/miniconda3 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 From 1c4ee171422ed8b0809489ede409bada09be37c5 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 10:36:09 -0700 Subject: [PATCH 064/156] temporarily try latest base-notebook --- common.sh | 1 - notebook/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/common.sh b/common.sh index 0b0d2d2..9bb29e4 100644 --- a/common.sh +++ b/common.sh @@ -35,7 +35,6 @@ chmod +x /usr/bin/cloud_sql_proxy # conda updates conda update -n base conda conda config --set channel_priority strict -cat ~/.condarc # filepath curating chmod +x /usr/bin/prepare.sh diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 12e61c6..8fdf25a 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,4 +1,4 @@ -FROM jupyter/base-notebook:python-3.7.3 +FROM jupyter/base-notebook:latest ARG DEBIAN_FRONTEND=noninteractive ## Add NB_USER to sudo From d6c85a6d7520ffce899234ba5561a20d7598bed4 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 10:42:22 -0700 Subject: [PATCH 065/156] move ca-certificates to before mc install --- common.sh | 1 - worker/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/common.sh b/common.sh index 9bb29e4..89fca44 100644 --- a/common.sh +++ b/common.sh @@ -5,7 +5,6 @@ apt-get update -y apt-get install -yq --no-install-recommends \ apt-utils \ bzip2 \ - ca-certificates \ curl \ lsb-release \ gnupg2 \ diff --git a/worker/Dockerfile b/worker/Dockerfile index 5a987a8..5e1dc9a 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -10,7 +10,7 @@ COPY prepare.sh /usr/bin/prepare.sh ########### RUN apt-get update --no-install-recommends -y -RUN apt-get install -yq --no-install-recommends wget +RUN apt-get install -yq --no-install-recommends wget ca-certificates # adapted from continuumio/miniconda3 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 From f0ea9443d417c6222999be2126f14ba1422ae948 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 12:46:15 -0700 Subject: [PATCH 066/156] try with geopandas in yaml --- base_environment.yml | 4 +++- common.sh | 4 ++-- notebook/Dockerfile | 4 +--- worker/Dockerfile | 6 +----- worker/environment.yml | 2 +- worker/prepare.sh | 8 ++++---- 6 files changed, 12 insertions(+), 16 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 4143b8c..2911c92 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -1,6 +1,7 @@ name: base channels: - conda-forge + - conda-forge/label/dev dependencies: - beautifulsoup4 - bokeh @@ -24,6 +25,7 @@ dependencies: - gcsfs - gdal - geoalchemy2 + - geopandas=1.0.0.dev - geopy - geotiff - geoviews @@ -49,8 +51,8 @@ dependencies: - pandas - pip - plotly - - pyshp - python=3.6 + - pyshp - python-blosc - python-snappy - pyviz_comms diff --git a/common.sh b/common.sh index 89fca44..f42c10a 100644 --- a/common.sh +++ b/common.sh @@ -1,4 +1,5 @@ #!/bin/sh +export CONDA_VERSION=4.7.12 # install apt-get packages apt-get update -y @@ -32,8 +33,7 @@ wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/clou chmod +x /usr/bin/cloud_sql_proxy # conda updates -conda update -n base conda -conda config --set channel_priority strict +conda install -n base conda=$CONDA_VERSION --no-pin # filepath curating chmod +x /usr/bin/prepare.sh diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 8fdf25a..1a791e0 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -33,9 +33,6 @@ COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json # update environemnt with common packages across worker and nb RUN conda env update -f /home/jovyan/base_environment.yml -# update with cythonized geopandas -RUN conda install --channel conda-forge/label/dev geopandas - # update environment with nb-specific packages RUN conda env update -f /home/jovyan/environment.yml @@ -70,6 +67,7 @@ RUN conda clean --all -y -f ## prepare container +USER $NB_USER WORKDIR $HOME ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] CMD ["start.sh jupyter lab"] diff --git a/worker/Dockerfile b/worker/Dockerfile index 5e1dc9a..6e42d41 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -36,10 +36,7 @@ COPY base_environment.yml /home/jovyan/base_environment.yml COPY environment.yml /home/jovyan/environment.yml # create environemnt with common packages across worker and nb -RUN conda env create -f /home/jovyan/base_environment.yml -n worker - -# update with cythonized geopandas -RUN conda install -n worker --channel conda-forge/label/dev geopandas +RUN conda env create -f /home/jovyan/base_environment.yml # update environment with worker-specific packages RUN conda env update -f /home/jovyan/environment.yml @@ -50,7 +47,6 @@ RUN gcloud components install kubectl ## clean up -USER root RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f diff --git a/worker/environment.yml b/worker/environment.yml index 2b02ddf..cb4b232 100644 --- a/worker/environment.yml +++ b/worker/environment.yml @@ -1,4 +1,4 @@ -name: worker +name: base channels: - conda-forge dependencies: diff --git a/worker/prepare.sh b/worker/prepare.sh index 6bf35b6..af09178 100644 --- a/worker/prepare.sh +++ b/worker/prepare.sh @@ -4,19 +4,19 @@ set -x if [[ -e "/opt/app/environment.yml" ]]; then echo "environment.yml found. Installing packages" - /opt/conda/bin/conda env update -n worker -f /opt/app/environment.yml + /opt/conda/bin/conda env update -f /opt/app/environment.yml else echo "no environment.yml" fi if [[ "$EXTRA_CONDA_PACKAGES" ]]; then echo "EXTRA_CONDA_PACKAGES environment variable found. Installing." - /opt/conda/bin/conda install -n worker --yes $EXTRA_CONDA_PACKAGES + /opt/conda/bin/conda install --yes $EXTRA_CONDA_PACKAGES fi if [[ "$EXTRA_PIP_PACKAGES" ]]; then echo "EXTRA_PIP_PACKAGES environment variable found. Installing". - /opt/conda/envs/worker/bin/pip install $EXTRA_PIP_PACKAGES + /opt/conda/bin/pip install $EXTRA_PIP_PACKAGES fi if [[ "$GCSFUSE_TOKENS" ]]; then @@ -50,4 +50,4 @@ if [[ "$GCLOUD_DEFAULT_TOKEN_FILE" ]]; then fi # Run extra commands -source activate worker && $@ +$@ From 6b830ab21fdcffa8350b7074612340c25b4b93a3 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 12:47:10 -0700 Subject: [PATCH 067/156] remove unnecessary packages from nb yaml --- notebook/environment.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/notebook/environment.yml b/notebook/environment.yml index 1b85d3f..a649f60 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -10,8 +10,6 @@ dependencies: - nbconvert - nodejs - nose - - jupyterhub - - jupyterlab - pytest - pytest-cov - pytest-runner From dbc5eca220838198530acc6bdd0e413ce17b1abd Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 12:54:11 -0700 Subject: [PATCH 068/156] fix env create v update in worker --- worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/Dockerfile b/worker/Dockerfile index 6e42d41..400951e 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -36,7 +36,7 @@ COPY base_environment.yml /home/jovyan/base_environment.yml COPY environment.yml /home/jovyan/environment.yml # create environemnt with common packages across worker and nb -RUN conda env create -f /home/jovyan/base_environment.yml +RUN conda env update -f /home/jovyan/base_environment.yml # update environment with worker-specific packages RUN conda env update -f /home/jovyan/environment.yml From 24733b21ee1f28df8eced780f9cfced2b36c666c Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 13:08:08 -0700 Subject: [PATCH 069/156] avoid 3.7 conflict w/ no-pin --- base_environment.yml | 2 ++ notebook/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index 2911c92..550a227 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -5,8 +5,10 @@ channels: dependencies: - beautifulsoup4 - bokeh + - boto3 - bqplot - bumpversion + - cachecontrol - cartopy - click - cloudpickle diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 1a791e0..eee38ac 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -34,7 +34,8 @@ COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json RUN conda env update -f /home/jovyan/base_environment.yml # update environment with nb-specific packages -RUN conda env update -f /home/jovyan/environment.yml +# (nopin needed b/c base-notebook pins python version) +RUN conda env update -f /home/jovyan/environment.yml --no-pin # create r environment RUN conda env create -f /home/jovyan/environment_r.yml From a145f00c988e2a5524e555276682313c99be06ce Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 13:25:41 -0700 Subject: [PATCH 070/156] fix no-pin --- notebook/Dockerfile | 10 ++++------ worker/Dockerfile | 4 ---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index eee38ac..bf1cfe8 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,6 +1,9 @@ FROM jupyter/base-notebook:latest ARG DEBIAN_FRONTEND=noninteractive +## get rid of conda pin file that causes update problems +RUN rm /opt/conda/conda-meta/pinned + ## Add NB_USER to sudo USER root RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook @@ -34,8 +37,7 @@ COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json RUN conda env update -f /home/jovyan/base_environment.yml # update environment with nb-specific packages -# (nopin needed b/c base-notebook pins python version) -RUN conda env update -f /home/jovyan/environment.yml --no-pin +RUN conda env update -f /home/jovyan/environment.yml # create r environment RUN conda env create -f /home/jovyan/environment_r.yml @@ -58,10 +60,6 @@ RUN jupyter serverextension enable --sys-prefix jupyter_server_proxy RUN jupyter serverextension enable --py --sys-prefix dask_labextension -## install kubectl -RUN gcloud components install kubectl - - ## clean up RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f diff --git a/worker/Dockerfile b/worker/Dockerfile index 400951e..12f4e91 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -42,10 +42,6 @@ RUN conda env update -f /home/jovyan/base_environment.yml RUN conda env update -f /home/jovyan/environment.yml -## install kubectl -RUN gcloud components install kubectl - - ## clean up RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f From 2cbe0bb420bed4e48c6408d584ed11507444a42d Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 15:09:09 -0700 Subject: [PATCH 071/156] patch geopandas and fix tini --- .travis.yml | 1 + base_environment.yml | 69 ++++++++++++++++++++-------------------- notebook/Dockerfile | 5 +++ notebook/environment.yml | 3 ++ patch_geopandas.sh | 5 +++ tmp.sh | 2 ++ worker/Dockerfile | 12 ++++--- worker/environment.yml | 6 ---- 8 files changed, 58 insertions(+), 45 deletions(-) create mode 100644 patch_geopandas.sh create mode 100644 tmp.sh delete mode 100644 worker/environment.yml diff --git a/.travis.yml b/.travis.yml index 075a157..009ea13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ install: - "cat notebook/worker-template.yml | grep image:" - "cp base_environment.yml $IMAGE_NAME/base_environment.yml" - "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh" +- "cp patch_geopandas.sh $IMAGE_NAME/patch_geopandas.sh && chmod +x $IMAGE_NAME/patch_geopandas.sh" - "cd $IMAGE_NAME" - "docker build -t rhodium/$IMAGE_NAME:$TRAVIS_COMMIT ." diff --git a/base_environment.yml b/base_environment.yml index 550a227..b44edf8 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -3,41 +3,39 @@ channels: - conda-forge - conda-forge/label/dev dependencies: - - beautifulsoup4 - - bokeh - - boto3 - - bqplot - - bumpversion - - cachecontrol - - cartopy - - click - - cloudpickle - - cytoolz - - dask - - dask-glm - - dask-kubernetes - - dask-ml - - datashader - - distributed - - esmf - - esmpy - - fastparquet - - fiona - - fusepy - - gcsfs - - gdal - - geoalchemy2 + - beautifulsoup4=4.8.0 + - bokeh=1.3.4 + - boto3=1.9.243 + - bqplot=0.11.8 + - bumpversion=0.5.3 + - cachecontrol=0.12.5 + - cartopy=0.17.0 + - click=7.0 + - cloudpickle=1.2.2 + - cytoolz=0.10.0 + - dask=2.5.2 + - dask-glm=0.2.0 + - dask-kubernetes=0.9.2 + - dask-ml=1.0.0 + - datashader=0.7.0 + - distributed=2.5.2 + - esmpy=7.1.0 + - fastparquet=0.3.2 + - fiona=1.8.7 + - fusepy=3.0.1 + - gcsfs=0.3.1 + - gdal=2.4.2 + - geoalchemy2=0.6.3 - geopandas=1.0.0.dev - - geopy - - geotiff - - geoviews - - git - - gitpython - - google-cloud-container - - google-cloud-storage - - holoviews - - icu - - ipdb + - geopy=1.20.0 + - geotiff=1.5.1 + - geoviews=1.6.3 + - git=2.23.0 + - gitpython=3.0.3 + - google-cloud-container=0.3.0 + - google-cloud-storage=1.20.0 + - holoviews=1.12.5 + - icu=64.2 - ipyleaflet - jedi - kubernetes @@ -53,6 +51,7 @@ dependencies: - pandas - pip - plotly + - polyline - python=3.6 - pyshp - python-blosc @@ -66,11 +65,13 @@ dependencies: - seaborn - shapely - statsmodels + - tini - toolz - tornado - tox - tqdm - urllib3 + - uritemplate - wheel - xarray - xesmf diff --git a/notebook/Dockerfile b/notebook/Dockerfile index bf1cfe8..f20368c 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -13,6 +13,7 @@ RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CO ## filepath curation COPY common.sh /home/jovyan/common.sh COPY prepare.sh /usr/bin/prepare.sh +COPY patch_geopandas.sh /home/jovyan/patch_geopandas.sh COPY base_environment.yml /home/jovyan/base_environment.yml RUN mkdir /pre-home && chown -R $NB_USER /pre-home @@ -64,6 +65,10 @@ RUN jupyter serverextension enable --py --sys-prefix dask_labextension RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f +# needed until cythonized geopandas updated for new pandas +RUN sudo /home/jovyan/patch_geopandas.sh +USER $NB_USER + ## prepare container USER $NB_USER diff --git a/notebook/environment.yml b/notebook/environment.yml index a649f60..8af464f 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -6,7 +6,9 @@ dependencies: - dask-labextension - flake8 - graphviz + - ipdb=0.12.2 - ipywidgets + - nano - nbconvert - nodejs - nose @@ -14,4 +16,5 @@ dependencies: - pytest-cov - pytest-runner - sphinx + - vim - widgetsnbextension diff --git a/patch_geopandas.sh b/patch_geopandas.sh new file mode 100644 index 0000000..d4c67c4 --- /dev/null +++ b/patch_geopandas.sh @@ -0,0 +1,5 @@ +#!/bin/bash +sed -i "s/from pandas.core.internals import Block, NonConsolidatableMixIn, \ +BlockManager/from pandas.core.internals.blocks import Block, \ +NonCnsolidatableMixIn\nfrom pandas.core.internals.managers import \ +BlockManager/g" /opt/conda/lib/python3.6/site-packages/geopandas/_block.py diff --git a/tmp.sh b/tmp.sh new file mode 100644 index 0000000..3f81661 --- /dev/null +++ b/tmp.sh @@ -0,0 +1,2 @@ +#!/bin/bash +$@ diff --git a/worker/Dockerfile b/worker/Dockerfile index 12f4e91..ed090d0 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -6,6 +6,7 @@ ARG DEBIAN_FRONTEND=noninteractive ## filepath curation COPY common.sh /home/jovyan/common.sh COPY prepare.sh /usr/bin/prepare.sh +COPY patch_geopandas.sh /home/jovyan/patch_geopandas.sh ########### @@ -33,21 +34,22 @@ RUN bash /home/jovyan/common.sh USER $NB_USER COPY add_service_creds.py /usr/bin/add_service_creds.py COPY base_environment.yml /home/jovyan/base_environment.yml -COPY environment.yml /home/jovyan/environment.yml # create environemnt with common packages across worker and nb RUN conda env update -f /home/jovyan/base_environment.yml -# update environment with worker-specific packages -RUN conda env update -f /home/jovyan/environment.yml - ## clean up RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f +# needed until cythonized geopandas updated for new pandas +USER root +RUN sudo /home/jovyan/patch_geopandas.sh +USER $NB_USER + ## prepare container ENV OMP_NUM_THREADS=1 ENV DASK_TICK_MAXIMUM_DELAY=5s -ENTRYPOINT ["/usr/local/bin/dumb-init", "/usr/bin/prepare.sh"] +ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] diff --git a/worker/environment.yml b/worker/environment.yml deleted file mode 100644 index cb4b232..0000000 --- a/worker/environment.yml +++ /dev/null @@ -1,6 +0,0 @@ -name: base -channels: - - conda-forge -dependencies: - - jupyter - - jupyter_client From 011b5f0f8b2a77dfde9a49c2270cb968a4dfecde Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 15:29:06 -0700 Subject: [PATCH 072/156] pin everything --- base_environment.yml | 95 ++++++++++++++++++---------------------- notebook/environment.yml | 32 +++++++------- 2 files changed, 60 insertions(+), 67 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index b44edf8..7162a6e 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -4,15 +4,12 @@ channels: - conda-forge/label/dev dependencies: - beautifulsoup4=4.8.0 - - bokeh=1.3.4 - boto3=1.9.243 - bqplot=0.11.8 - bumpversion=0.5.3 - cachecontrol=0.12.5 - cartopy=0.17.0 - click=7.0 - - cloudpickle=1.2.2 - - cytoolz=0.10.0 - dask=2.5.2 - dask-glm=0.2.0 - dask-kubernetes=0.9.2 @@ -36,54 +33,48 @@ dependencies: - google-cloud-storage=1.20.0 - holoviews=1.12.5 - icu=64.2 - - ipyleaflet - - jedi - - kubernetes - - lz4 - - make - - matplotlib - - nb_conda_kernels - - ncurses - - netcdf-fortran - - netcdf4 - - numba - - numcodecs - - pandas - - pip - - plotly - - polyline + - ipyleaflet=0.11.2 + - jedi=0.15.1 + - kubernetes=1.16.1 + - lz4=2.2.1 + - make=4.2.1 + - matplotlib=3.1.1 + - nb_conda_kernels=2.2.2 + - ncurses=6.1 + - netcdf-fortran=4.4.5 + - netcdf4=1.5.1.2 + - numba=0.45.1 + - numcodecs=0.6.3 + - pandas=0.25.1 + - pip=19.2.3 + - plotly=4.1.1 + - polyline=1.4.0 - python=3.6 - - pyshp - - python-blosc - - python-snappy - - pyviz_comms - - PyYAML - - rasterio - - scikit-image - - scikit-learn - - scipy - - seaborn - - shapely - - statsmodels - - tini - - toolz - - tornado - - tox - - tqdm - - urllib3 - - uritemplate - - wheel - - xarray - - xesmf - - xlrd - - zarr - - zeromq - - zict + - pyshp=2.1.0 + - python-blosc=1.8.1 + - python-snappy=0.5.4 + - pyviz_comms=0.7.2 + - PyYAML=5.1.2 + - rasterio=1.0.28 + - scikit-image=0.15.0 + - scikit-learn=0.21.3 + - scipy=1.3.1 + - seaborn=0.9.0 + - shapely=1.6.4 + - statsmodels=0.10.1 + - tini=0.18.0 + - uritemplate=3.0.0 + - xarray=0.13.0 + - xesmf=0.2.0 + - xlrd=1.2.0 + - zarr=2.3.2 + - zeromq=4.3.2 + - zict=1.0.0 - pip: - - mapbox - - py-noaa - - sidecar - - rhg_compute_tools - - impactlab-tools - - climate-toolbox - - parameterize_jobs + - mapbox==0.18.0 + - py-noaa==1.0 + - sidecar==0.3.0 + - climate-toolbox==0.1.5 + - impactlab-tools==0.3.1 + - parameterize_jobs==0.1.1 + - rhg_compute_tools==0.2.0 diff --git a/notebook/environment.yml b/notebook/environment.yml index 8af464f..81fb3e2 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -2,19 +2,21 @@ name: base channels: - conda-forge dependencies: - - coverage - - dask-labextension - - flake8 - - graphviz + - coverage=4.5.4 + - dask-labextension=1.0.3 + - flake8=3.7.8 + - graphviz=2.40.1 - ipdb=0.12.2 - - ipywidgets - - nano - - nbconvert - - nodejs - - nose - - pytest - - pytest-cov - - pytest-runner - - sphinx - - vim - - widgetsnbextension + - ipywidgets=7.5.1 + - nano=2.9.8 + - nbconvert=5.6.0 + - nodejs=12.11.1 + - nose=1.3.7 + - pytest=5.2.1 + - pytest-cov=2.8.1 + - pytest-runner=5.1 + - sphinx=2.2.0 + - tox=3.14.0 + - tqdm=4.36.1 + - vim=8.1.1343 + - wheel=0.33.6 From 571fbf1cc9ef2d4e1407fdba766b6ef80044e59e Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 16:16:01 -0700 Subject: [PATCH 073/156] remove extraneous env var from worker prepare --- worker/prepare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/worker/prepare.sh b/worker/prepare.sh index af09178..8e198b2 100644 --- a/worker/prepare.sh +++ b/worker/prepare.sh @@ -45,8 +45,8 @@ if [[ "$SQL_TOKEN" ]]; then fi; fi -if [[ "$GCLOUD_DEFAULT_TOKEN_FILE" ]]; then - gcloud auth activate-service-account --key-file $GCLOUD_DEFAULT_TOKEN_FILE; +if [[ "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then + gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS; fi # Run extra commands From 15d8cfa2523b71f6c105a00e927e6488bbf8ff9d Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 16:29:21 -0700 Subject: [PATCH 074/156] fix dumb geopandas patch typo --- patch_geopandas.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patch_geopandas.sh b/patch_geopandas.sh index d4c67c4..23be059 100644 --- a/patch_geopandas.sh +++ b/patch_geopandas.sh @@ -1,5 +1,5 @@ #!/bin/bash sed -i "s/from pandas.core.internals import Block, NonConsolidatableMixIn, \ BlockManager/from pandas.core.internals.blocks import Block, \ -NonCnsolidatableMixIn\nfrom pandas.core.internals.managers import \ +NonConslidatableMixIn\nfrom pandas.core.internals.managers import \ BlockManager/g" /opt/conda/lib/python3.6/site-packages/geopandas/_block.py From 316f1df5fb7a87fefac1cdc2d71b41eec7a6c64a Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 16:35:15 -0700 Subject: [PATCH 075/156] add conda fort and c compilers --- base_environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base_environment.yml b/base_environment.yml index 7162a6e..c0333dc 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -10,6 +10,7 @@ dependencies: - cachecontrol=0.12.5 - cartopy=0.17.0 - click=7.0 + - c-compiler=1.0.4 - dask=2.5.2 - dask-glm=0.2.0 - dask-kubernetes=0.9.2 @@ -19,6 +20,7 @@ dependencies: - esmpy=7.1.0 - fastparquet=0.3.2 - fiona=1.8.7 + - fortran-compiler=1.0.4 - fusepy=3.0.1 - gcsfs=0.3.1 - gdal=2.4.2 From c0816612816698f7c5087b712922a36efac9f541 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 16:39:11 -0700 Subject: [PATCH 076/156] add all conda compilers package --- base_environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index c0333dc..62d41fe 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -10,7 +10,7 @@ dependencies: - cachecontrol=0.12.5 - cartopy=0.17.0 - click=7.0 - - c-compiler=1.0.4 + - compilers=1.0.4 - dask=2.5.2 - dask-glm=0.2.0 - dask-kubernetes=0.9.2 @@ -20,7 +20,6 @@ dependencies: - esmpy=7.1.0 - fastparquet=0.3.2 - fiona=1.8.7 - - fortran-compiler=1.0.4 - fusepy=3.0.1 - gcsfs=0.3.1 - gdal=2.4.2 From 6bd547ef1ee50f18abd4aae8ba8cdf11f12238fb Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 17:43:46 -0700 Subject: [PATCH 077/156] add openssh --- notebook/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/notebook/environment.yml b/notebook/environment.yml index 81fb3e2..060cac8 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -12,6 +12,7 @@ dependencies: - nbconvert=5.6.0 - nodejs=12.11.1 - nose=1.3.7 + - openssh=7.9pl - pytest=5.2.1 - pytest-cov=2.8.1 - pytest-runner=5.1 From bfd95949de0c38c74491730f93ad8a844f2eee7f Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 18:18:36 -0700 Subject: [PATCH 078/156] open openssh and figure out user permissions --- notebook/Dockerfile | 25 ++++++++++++------------- notebook/environment.yml | 2 +- worker/Dockerfile | 5 +---- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index f20368c..f19b337 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -4,30 +4,31 @@ ARG DEBIAN_FRONTEND=noninteractive ## get rid of conda pin file that causes update problems RUN rm /opt/conda/conda-meta/pinned + ## Add NB_USER to sudo USER root RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook RUN sed -ri "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers +USER $NB_USER ## filepath curation -COPY common.sh /home/jovyan/common.sh -COPY prepare.sh /usr/bin/prepare.sh -COPY patch_geopandas.sh /home/jovyan/patch_geopandas.sh -COPY base_environment.yml /home/jovyan/base_environment.yml - -RUN mkdir /pre-home && chown -R $NB_USER /pre-home +RUN sudo mkdir /pre-home && sudo chown -R $NB_USER /pre-home COPY worker-template.yml /pre-home COPY add_service_creds.py /pre-home COPY run_sql_proxy.py /pre-home COPY config.yaml /pre-home COPY worker-template.yml /pre-home +COPY common.sh /home/jovyan/common.sh +COPY prepare.sh /usr/bin/prepare.sh +COPY patch_geopandas.sh /home/jovyan/patch_geopandas.sh +COPY base_environment.yml /home/jovyan/base_environment.yml -## perform a bunch of common actions -RUN bash /home/jovyan/common.sh -RUN chown -R $NB_USER /gcs +## perform a bunch of common actions +RUN sudo bash /home/jovyan/common.sh +RUN sudo chown -R $NB_USER /gcs ## set up python env COPY environment.yml /home/jovyan/environment.yml @@ -62,16 +63,14 @@ RUN jupyter serverextension enable --py --sys-prefix dask_labextension ## clean up -RUN rm -rf /var/lib/apt/lists/* +RUN sudo rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f # needed until cythonized geopandas updated for new pandas -RUN sudo /home/jovyan/patch_geopandas.sh -USER $NB_USER +RUN /home/jovyan/patch_geopandas.sh ## prepare container -USER $NB_USER WORKDIR $HOME ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] CMD ["start.sh jupyter lab"] diff --git a/notebook/environment.yml b/notebook/environment.yml index 060cac8..2ef139d 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -12,7 +12,7 @@ dependencies: - nbconvert=5.6.0 - nodejs=12.11.1 - nose=1.3.7 - - openssh=7.9pl + - openssh - pytest=5.2.1 - pytest-cov=2.8.1 - pytest-runner=5.1 diff --git a/worker/Dockerfile b/worker/Dockerfile index ed090d0..7a6b994 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -31,7 +31,6 @@ RUN bash /home/jovyan/common.sh ## set up python env -USER $NB_USER COPY add_service_creds.py /usr/bin/add_service_creds.py COPY base_environment.yml /home/jovyan/base_environment.yml @@ -44,9 +43,7 @@ RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f # needed until cythonized geopandas updated for new pandas -USER root -RUN sudo /home/jovyan/patch_geopandas.sh -USER $NB_USER +RUN /home/jovyan/patch_geopandas.sh ## prepare container From 9d86abd01666fb42bc270bb58567da60fef52afa Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 18:33:43 -0700 Subject: [PATCH 079/156] retry permissions and unpin r temporarily --- common.sh | 3 --- notebook/Dockerfile | 2 ++ notebook/environment_r.yml | 40 +++++++++++++++++++------------------- worker/Dockerfile | 1 + 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/common.sh b/common.sh index f42c10a..0678c98 100644 --- a/common.sh +++ b/common.sh @@ -32,9 +32,6 @@ apt-get clean wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/bin/cloud_sql_proxy chmod +x /usr/bin/cloud_sql_proxy -# conda updates -conda install -n base conda=$CONDA_VERSION --no-pin - # filepath curating chmod +x /usr/bin/prepare.sh mkdir /gcs diff --git a/notebook/Dockerfile b/notebook/Dockerfile index f19b337..7d18687 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -30,7 +30,9 @@ COPY base_environment.yml /home/jovyan/base_environment.yml RUN sudo bash /home/jovyan/common.sh RUN sudo chown -R $NB_USER /gcs + ## set up python env +RUN conda install -n base conda=$CONDA_VERSION --no-pin COPY environment.yml /home/jovyan/environment.yml COPY environment_r.yml /home/jovyan/environment_r.yml COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json diff --git a/notebook/environment_r.yml b/notebook/environment_r.yml index 8989b16..417f459 100644 --- a/notebook/environment_r.yml +++ b/notebook/environment_r.yml @@ -3,23 +3,23 @@ channels: - conda-forge - r dependencies: - - rpy2=2.8* - - r-base=3.4.1 - - r-irkernel=0.8* - - r-plyr=1.8* - - r-devtools=1.13* - - r-tidyverse=1.1* - - r-shiny=1.0* - - r-rmarkdown=1.8* - - r-forecast=8.2* - - r-rsqlite=2.0* - - r-reshape2=1.4* - - r-nycflights13=0.2* - - r-caret=6.0* - - r-rcurl=1.95* - - r-crayon=1.3* - - r-randomforest=4.6* - - r-htmltools=0.3* - - r-sparklyr=0.7* - - r-htmlwidgets=1.0* - - r-hexbin=1.27* + - rpy2 + - r-base + - r-irkernel + - r-plyr + - r-devtools + - r-tidyverse + - r-shiny + - r-rmarkdown + - r-forecast + - r-rsqlite + - r-reshape2 + - r-nycflights13 + - r-caret + - r-rcurl + - r-crayon + - r-randomforest + - r-htmltools + - r-sparklyr + - r-htmlwidgets + - r-hexbin diff --git a/worker/Dockerfile b/worker/Dockerfile index 7a6b994..f0b2d8b 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -31,6 +31,7 @@ RUN bash /home/jovyan/common.sh ## set up python env +RUN conda install -n base conda=$CONDA_VERSION --no-pin COPY add_service_creds.py /usr/bin/add_service_creds.py COPY base_environment.yml /home/jovyan/base_environment.yml From 663b03952b05aa761467d104fdd6478dff686b3e Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 18:35:22 -0700 Subject: [PATCH 080/156] another patch typo --- patch_geopandas.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patch_geopandas.sh b/patch_geopandas.sh index 23be059..18f51d6 100644 --- a/patch_geopandas.sh +++ b/patch_geopandas.sh @@ -1,5 +1,5 @@ #!/bin/bash sed -i "s/from pandas.core.internals import Block, NonConsolidatableMixIn, \ BlockManager/from pandas.core.internals.blocks import Block, \ -NonConslidatableMixIn\nfrom pandas.core.internals.managers import \ +NonConsolidatableMixIn\nfrom pandas.core.internals.managers import \ BlockManager/g" /opt/conda/lib/python3.6/site-packages/geopandas/_block.py From 720eefe2993efde101b6785efaa5ae041e176bc0 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 19:06:00 -0700 Subject: [PATCH 081/156] conda version fix --- common.sh | 2 -- notebook/Dockerfile | 5 ++++- notebook/environment.yml | 3 +-- worker/Dockerfile | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common.sh b/common.sh index 0678c98..117a684 100644 --- a/common.sh +++ b/common.sh @@ -1,6 +1,4 @@ #!/bin/sh -export CONDA_VERSION=4.7.12 - # install apt-get packages apt-get update -y apt-get install -yq --no-install-recommends \ diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 7d18687..5894b0e 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -30,9 +30,12 @@ COPY base_environment.yml /home/jovyan/base_environment.yml RUN sudo bash /home/jovyan/common.sh RUN sudo chown -R $NB_USER /gcs +# more apt-get +RUN apt-get install -yq --no-install-recommends vim && apt-get clean + ## set up python env -RUN conda install -n base conda=$CONDA_VERSION --no-pin +RUN conda install -n base conda=4.7.12 --no-pin COPY environment.yml /home/jovyan/environment.yml COPY environment_r.yml /home/jovyan/environment_r.yml COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json diff --git a/notebook/environment.yml b/notebook/environment.yml index 2ef139d..51e8e30 100644 --- a/notebook/environment.yml +++ b/notebook/environment.yml @@ -12,12 +12,11 @@ dependencies: - nbconvert=5.6.0 - nodejs=12.11.1 - nose=1.3.7 - - openssh + - openssh=7.9p1 - pytest=5.2.1 - pytest-cov=2.8.1 - pytest-runner=5.1 - sphinx=2.2.0 - tox=3.14.0 - tqdm=4.36.1 - - vim=8.1.1343 - wheel=0.33.6 diff --git a/worker/Dockerfile b/worker/Dockerfile index f0b2d8b..7e63966 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -31,7 +31,7 @@ RUN bash /home/jovyan/common.sh ## set up python env -RUN conda install -n base conda=$CONDA_VERSION --no-pin +RUN conda install -n base conda=4.7.12 --no-pin COPY add_service_creds.py /usr/bin/add_service_creds.py COPY base_environment.yml /home/jovyan/base_environment.yml From 3f52deca6297dd55da2af0303f4222cb38b38506 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 19:08:06 -0700 Subject: [PATCH 082/156] get rid of unneeded no-pin flag --- notebook/Dockerfile | 2 +- worker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 5894b0e..d047d03 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get install -yq --no-install-recommends vim && apt-get clean ## set up python env -RUN conda install -n base conda=4.7.12 --no-pin +RUN conda install -n base conda=4.7.12 COPY environment.yml /home/jovyan/environment.yml COPY environment_r.yml /home/jovyan/environment_r.yml COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json diff --git a/worker/Dockerfile b/worker/Dockerfile index 7e63966..44aa705 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -31,7 +31,7 @@ RUN bash /home/jovyan/common.sh ## set up python env -RUN conda install -n base conda=4.7.12 --no-pin +RUN conda install -n base conda=4.7.12 COPY add_service_creds.py /usr/bin/add_service_creds.py COPY base_environment.yml /home/jovyan/base_environment.yml From 55be974a86ff6fbedbb7dc75216d06001fda3576 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 19:15:03 -0700 Subject: [PATCH 083/156] sudo the apt-get --- notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index d047d03..81e6287 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -31,7 +31,7 @@ RUN sudo bash /home/jovyan/common.sh RUN sudo chown -R $NB_USER /gcs # more apt-get -RUN apt-get install -yq --no-install-recommends vim && apt-get clean +RUN sudo apt-get install -yq --no-install-recommends vim && apt-get clean ## set up python env From eef7649b3cb15290ce69650edd6772b468860688 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 19:20:22 -0700 Subject: [PATCH 084/156] repin r packages --- notebook/environment_r.yml | 39 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/notebook/environment_r.yml b/notebook/environment_r.yml index 417f459..3d97a4a 100644 --- a/notebook/environment_r.yml +++ b/notebook/environment_r.yml @@ -3,23 +3,22 @@ channels: - conda-forge - r dependencies: - - rpy2 - - r-base - - r-irkernel - - r-plyr - - r-devtools - - r-tidyverse - - r-shiny - - r-rmarkdown - - r-forecast - - r-rsqlite - - r-reshape2 - - r-nycflights13 - - r-caret - - r-rcurl - - r-crayon - - r-randomforest - - r-htmltools - - r-sparklyr - - r-htmlwidgets - - r-hexbin + - rpy2=3.1.0 + - r-base=3.6.1 + - r-irkernel=1.0.2 + - r-plyr=1.8.4 + - r-devtools=2.2.1 + - r-tidyverse=1.2.1 + - r-shiny=1.3.2 + - r-rmarkdown=1.16 + - r-forecast=8.9 + - r-rsqlite=2.1.2 + - r-reshape2=1.4.3 + - r-nycflights13=1.0.1 + - r-caret=6.0_84 + - r-rcurl=1.95_4.12 + - r-crayon=1.3.4 + - r-randomforest=4.6_1 + - r-sparklyr=1.0.3 + - r-htmlwidgets=1.5 + - r-hexbin=1.27.3 From f6e903aa59429aeed7a944da34f9b97cc14020f3 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 19:22:51 -0700 Subject: [PATCH 085/156] sudo the apt clean --- notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 81e6287..f11011c 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -31,7 +31,7 @@ RUN sudo bash /home/jovyan/common.sh RUN sudo chown -R $NB_USER /gcs # more apt-get -RUN sudo apt-get install -yq --no-install-recommends vim && apt-get clean +RUN sudo apt-get install -yq --no-install-recommends vim && sudo apt-get clean ## set up python env From d48911d47226e82fb43bf136343cd7ededb15f8f Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 6 Oct 2019 19:37:47 -0700 Subject: [PATCH 086/156] fix randomforest vers --- notebook/environment_r.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/environment_r.yml b/notebook/environment_r.yml index 3d97a4a..c4bca8e 100644 --- a/notebook/environment_r.yml +++ b/notebook/environment_r.yml @@ -18,7 +18,7 @@ dependencies: - r-caret=6.0_84 - r-rcurl=1.95_4.12 - r-crayon=1.3.4 - - r-randomforest=4.6_1 + - r-randomforest=4.6_14 - r-sparklyr=1.0.3 - r-htmlwidgets=1.5 - r-hexbin=1.27.3 From 1364e7c20d72a2493d4b5f4dc2cec34f0ec2b220 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 7 Oct 2019 23:19:19 -0700 Subject: [PATCH 087/156] set conda-forge --- notebook/Dockerfile | 3 ++- worker/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index f11011c..c1fe196 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -69,7 +69,8 @@ RUN jupyter serverextension enable --py --sys-prefix dask_labextension ## clean up RUN sudo rm -rf /var/lib/apt/lists/* -RUN conda clean --all -y -f +RUN conda clean --all -y -f && \ + conda config --add channels conda-forge # needed until cythonized geopandas updated for new pandas RUN /home/jovyan/patch_geopandas.sh diff --git a/worker/Dockerfile b/worker/Dockerfile index 44aa705..9857e51 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -41,7 +41,8 @@ RUN conda env update -f /home/jovyan/base_environment.yml ## clean up RUN rm -rf /var/lib/apt/lists/* -RUN conda clean --all -y -f +RUN conda clean --all -y -f && \ + conda config --add channels conda-forge # needed until cythonized geopandas updated for new pandas RUN /home/jovyan/patch_geopandas.sh From 6861ecb1e789c54edad032e22ff72f143bdc602f Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Wed, 9 Oct 2019 13:01:33 -0700 Subject: [PATCH 088/156] try specifying worker address in dask-worker --- notebook/worker-template.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml index 37f3c54..59eef60 100644 --- a/notebook/worker-template.yml +++ b/notebook/worker-template.yml @@ -2,15 +2,8 @@ metadata: spec: restartPolicy: Never containers: - - args: - - dask-worker - - --nthreads - - '1' - - --no-bokeh - - --memory-limit - - 11.5GB - - --death-timeout - - '60' + - args: [dask-worker, $(DASK_SCHEDULER_ADDRESS), --nthreads, '1', --no-bokeh, + --memory-limit, 11.5GB, --death-timeout, '60'] env: - name: GCSFUSE_BUCKET value: rhg-data From 7b7e4e930f26437d22d85af796881d93c3e797fa Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Wed, 23 Oct 2019 07:56:59 -0700 Subject: [PATCH 089/156] add a bunch of packages from cmip6 pangeo hack --- base_environment.yml | 47 +++++++++++++++++++++++++++----------------- notebook/Dockerfile | 1 + worker/Dockerfile | 1 + 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 62d41fe..3ac5b87 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -3,23 +3,24 @@ channels: - conda-forge - conda-forge/label/dev dependencies: - - beautifulsoup4=4.8.0 - - boto3=1.9.243 + - beautifulsoup4=4.8.1 + - boto3=1.10.0 - bqplot=0.11.8 - bumpversion=0.5.3 - cachecontrol=0.12.5 - cartopy=0.17.0 + - cftime=1.0.4 - click=7.0 - compilers=1.0.4 - - dask=2.5.2 + - dask=2.6.0 - dask-glm=0.2.0 - - dask-kubernetes=0.9.2 + - dask-kubernetes=0.10.0 - dask-ml=1.0.0 - - datashader=0.7.0 - - distributed=2.5.2 + - datashader=0.8.0 + - distributed=2.6.0 - esmpy=7.1.0 - fastparquet=0.3.2 - - fiona=1.8.7 + - fiona=1.8.9 - fusepy=3.0.1 - gcsfs=0.3.1 - gdal=2.4.2 @@ -27,47 +28,57 @@ dependencies: - geopandas=1.0.0.dev - geopy=1.20.0 - geotiff=1.5.1 - - geoviews=1.6.3 + - geoviews=1.6.5 - git=2.23.0 - - gitpython=3.0.3 + - gitpython=3.0.4 - google-cloud-container=0.3.0 - google-cloud-storage=1.20.0 - - holoviews=1.12.5 + - holoviews=1.12.6 + - h5netcdf=0.7.4 - icu=64.2 - - ipyleaflet=0.11.2 + - intake-esm=2019.10.1 + - ipyleaflet=0.11.4 - jedi=0.15.1 - - kubernetes=1.16.1 + - kubernetes=1.16.2 + - lapack=3.6.1 - lz4=2.2.1 - make=4.2.1 - matplotlib=3.1.1 - nb_conda_kernels=2.2.2 + - nc-time-axis=1.2.0 - ncurses=6.1 - netcdf-fortran=4.4.5 - netcdf4=1.5.1.2 - numba=0.45.1 - numcodecs=0.6.3 - - pandas=0.25.1 - - pip=19.2.3 - - plotly=4.1.1 + - pandas=0.25.2 + - pip=19.3.1 + - plotly=4.2.1 - polyline=1.4.0 - - python=3.6 + - pyinterp=0.0.6 - pyshp=2.1.0 + - python=3.6 - python-blosc=1.8.1 - python-snappy=0.5.4 - pyviz_comms=0.7.2 - PyYAML=5.1.2 - rasterio=1.0.28 - - scikit-image=0.15.0 + - regionmask=0.4.0 + - scikit-image=0.16.2 - scikit-learn=0.21.3 - scipy=1.3.1 - seaborn=0.9.0 - shapely=1.6.4 + - sparse=0.8.0 - statsmodels=0.10.1 - tini=0.18.0 - uritemplate=3.0.0 - - xarray=0.13.0 + - xarray=0.14.0 - xesmf=0.2.0 + - xgcm=0.2.0 + - xhistogram=0.1.1 - xlrd=1.2.0 + - xrft=0.2.0 - zarr=2.3.2 - zeromq=4.3.2 - zict=1.0.0 diff --git a/notebook/Dockerfile b/notebook/Dockerfile index c1fe196..bab6b96 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -70,6 +70,7 @@ RUN jupyter serverextension enable --py --sys-prefix dask_labextension ## clean up RUN sudo rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f && \ + conda config --add channels conda-forge/label/dev && \ conda config --add channels conda-forge # needed until cythonized geopandas updated for new pandas diff --git a/worker/Dockerfile b/worker/Dockerfile index 9857e51..a5dd671 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -42,6 +42,7 @@ RUN conda env update -f /home/jovyan/base_environment.yml ## clean up RUN rm -rf /var/lib/apt/lists/* RUN conda clean --all -y -f && \ + conda config --add channels conda-forge/label/dev && \ conda config --add channels conda-forge # needed until cythonized geopandas updated for new pandas From a8296753f4614bd67335d72ac7fdaaca57cc9620 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Wed, 23 Oct 2019 08:00:18 -0700 Subject: [PATCH 090/156] drop extraneous prepare.sh code --- notebook/prepare.sh | 22 ---------------------- worker/prepare.sh | 7 ------- 2 files changed, 29 deletions(-) diff --git a/notebook/prepare.sh b/notebook/prepare.sh index 0608e34..a79c1d5 100644 --- a/notebook/prepare.sh +++ b/notebook/prepare.sh @@ -11,28 +11,6 @@ sudo rm /pre-home/config.yaml echo "Copy files from pre-load directory into home" cp --update -r -v /pre-home/. /home/jovyan -if [[ -e "/opt/app/environment.yml" ]]; then - echo "environment.yml found. Installing packages"; - /opt/conda/bin/conda env update -f /opt/app/environment.yml; -else - echo "no environment.yml"; -fi - -if [[ "$EXTRA_CONDA_PACKAGES" ]]; then - echo "EXTRA_CONDA_PACKAGES environment variable found. Installing."; - /opt/conda/bin/conda install $EXTRA_CONDA_PACKAGES; -fi - -if [[ "$EXTRA_PIP_PACKAGES" ]]; then - echo "EXTRA_PIP_PACKAGES environment variable found. Installing".; - /opt/conda/bin/pip install $EXTRA_PIP_PACKAGES; -fi - -if [[ -e "/home/jovyan/conda_environment.yml" ]]; then - echo "installing conda env"; - /opt/conda/bin/conda env create -f /home/jovyan/conda_environment.yml; -fi - # mirror directory used on workers sudo mkdir -p /opt/gcsfuse_tokens/ mkdir -p /home/jovyan/service-account-credentials/ diff --git a/worker/prepare.sh b/worker/prepare.sh index 8e198b2..90ebc0e 100644 --- a/worker/prepare.sh +++ b/worker/prepare.sh @@ -2,13 +2,6 @@ set -x -if [[ -e "/opt/app/environment.yml" ]]; then - echo "environment.yml found. Installing packages" - /opt/conda/bin/conda env update -f /opt/app/environment.yml -else - echo "no environment.yml" -fi - if [[ "$EXTRA_CONDA_PACKAGES" ]]; then echo "EXTRA_CONDA_PACKAGES environment variable found. Installing." /opt/conda/bin/conda install --yes $EXTRA_CONDA_PACKAGES From fb18e11a25bb32259ecbfb38d1b590a79627ab7d Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Wed, 23 Oct 2019 08:14:40 -0700 Subject: [PATCH 091/156] switch to latest intake-esm --- base_environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index 3ac5b87..02d2f80 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -36,7 +36,6 @@ dependencies: - holoviews=1.12.6 - h5netcdf=0.7.4 - icu=64.2 - - intake-esm=2019.10.1 - ipyleaflet=0.11.4 - jedi=0.15.1 - kubernetes=1.16.2 @@ -55,6 +54,7 @@ dependencies: - pip=19.3.1 - plotly=4.2.1 - polyline=1.4.0 + - pygeos=0.4 - pyinterp=0.0.6 - pyshp=2.1.0 - python=3.6 @@ -90,3 +90,4 @@ dependencies: - impactlab-tools==0.3.1 - parameterize_jobs==0.1.1 - rhg_compute_tools==0.2.0 + - git+https://github.com/NCAR/intake-esm.git From 1659b959909ba0c1cc5c51ac8c05c5b3e884ccc1 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Wed, 23 Oct 2019 15:40:08 -0700 Subject: [PATCH 092/156] add unzip to images --- base_environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/base_environment.yml b/base_environment.yml index 02d2f80..3f1cb38 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -72,6 +72,7 @@ dependencies: - sparse=0.8.0 - statsmodels=0.10.1 - tini=0.18.0 + - unzip=6.0 - uritemplate=3.0.0 - xarray=0.14.0 - xesmf=0.2.0 From 899b043cad5ab10ca618beeab7110fea08bcdadf Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 28 Oct 2019 18:46:09 -0400 Subject: [PATCH 093/156] fixing up small things --- notebook/worker-template.yml | 2 +- tmp.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 tmp.sh diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml index 59eef60..c1686b6 100644 --- a/notebook/worker-template.yml +++ b/notebook/worker-template.yml @@ -7,7 +7,7 @@ spec: env: - name: GCSFUSE_BUCKET value: rhg-data - image: rhodium/worker:dev-update-packages + image: rhodium/worker:WILL_BE_REPLACED_BY_TRAVIS name: dask-worker securityContext: capabilities: diff --git a/tmp.sh b/tmp.sh deleted file mode 100644 index 3f81661..0000000 --- a/tmp.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -$@ From 6d35766b6b86daedbbb55115682fcf6070ce1a81 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Tue, 29 Oct 2019 09:06:44 -0700 Subject: [PATCH 094/156] document geopandas patch --- patch_geopandas.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/patch_geopandas.sh b/patch_geopandas.sh index 18f51d6..05ce4cc 100644 --- a/patch_geopandas.sh +++ b/patch_geopandas.sh @@ -1,4 +1,11 @@ #!/bin/bash + +# manually edit geopandas/label/dev to match current pandas deployment +# see https://github.com/geopandas/geopandas/issues/473#issuecomment-465060849 +# should be removed if geopandas core becomes vectorized, whether the +# cythonized branch gets merged in or the geometry calculations are handled +# by pyGEOS, either through shapely or replacing shapely. See +# https://github.com/Toblerity/Shapely/issues/782. sed -i "s/from pandas.core.internals import Block, NonConsolidatableMixIn, \ BlockManager/from pandas.core.internals.blocks import Block, \ NonConsolidatableMixIn\nfrom pandas.core.internals.managers import \ From f98c6dc1236271359c016867bb974f988f4ddebe Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 07:41:09 -0800 Subject: [PATCH 095/156] dockerfile typos --- notebook/Dockerfile | 2 +- worker/Dockerfile | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 4002a62..704e024 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -34,7 +34,7 @@ COPY overrides.json /opt/conda/share/jupyter/lab/settings ## perform a bunch of common actions -RUN sudo bash /pre-home/common.sh +RUN sudo bash /tempdir/common.sh RUN sudo chown -R $NB_USER /gcs diff --git a/worker/Dockerfile b/worker/Dockerfile index 6c03583..465c92c 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -2,17 +2,6 @@ FROM ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c ARG DEBIAN_FRONTEND=noninteractive -## filepath curation -RUN mkdir /opt/conda/specs -COPY base_environment.yml /opt/conda/specs - -RUN mkdir /tempdir -COPY common.sh /tempdir -COPY patch_geopandas.sh /tempdir - -COPY add_service_creds.py /usr/bin -COPY prepare.sh /usr/bin - ########### RUN apt-get update --no-install-recommends -y @@ -31,13 +20,24 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86 ########### +## filepath curation +RUN mkdir /opt/conda/specs +COPY base_environment.yml /opt/conda/specs + +RUN mkdir /tempdir +COPY common.sh /tempdir +COPY patch_geopandas.sh /tempdir + +COPY add_service_creds.py /usr/bin +COPY prepare.sh /usr/bin + + ## perform a bunch of common actions RUN bash /tempdir/common.sh + ## set up python env RUN conda update -n base conda - -# create environemnt with common packages across worker and nb RUN conda env update -f /opt/conda/specs/base_environment.yml From 48c0c3850b8190c5314a7edb33fb8e233f0a5afc Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 07:57:43 -0800 Subject: [PATCH 096/156] update from conda-forge --- notebook/Dockerfile | 13 +++++++++---- worker/Dockerfile | 9 ++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 704e024..78700b3 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -42,8 +42,13 @@ RUN sudo chown -R $NB_USER /gcs RUN sudo apt-get install -yq --no-install-recommends vim && sudo apt-get clean +# set up conda channels +RUN conda config --add channels conda-forge/label/dev && \ + conda config --add channels conda-forge + + ## set up conda -RUN conda install -n base conda +RUN conda update -n base conda # update environemnt with common packages across worker and nb RUN conda env update -f /opt/conda/specs/base_environment.yml @@ -55,6 +60,7 @@ RUN conda env update -f /opt/conda/specs/notebook_environment.yml RUN conda env create -f /opt/conda/specs/r_environment.yml +## Set up jupyter lab extensions RUN jupyter labextension update --all && \ jupyter labextension install \ @jupyterlab/hub-extension \ @@ -76,11 +82,10 @@ RUN jupyter serverextension enable --py --sys-prefix dask_labextension ## needed until cythonized geopandas updated for new pandas RUN /tempdir/patch_geopandas.sh + ## clean up RUN sudo rm -rf /var/lib/apt/lists/* /tempdir -RUN conda clean --all -y -f && \ - conda config --add channels conda-forge/label/dev && \ - conda config --add channels conda-forge +RUN conda clean --all -f ## prepare container diff --git a/worker/Dockerfile b/worker/Dockerfile index 465c92c..94ff25e 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -20,6 +20,11 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86 ########### +## set up conda channels +RUN conda config --add channels conda-forge/label/dev && \ + conda config --add channels conda-forge + + ## filepath curation RUN mkdir /opt/conda/specs COPY base_environment.yml /opt/conda/specs @@ -47,9 +52,7 @@ RUN /tempdir/patch_geopandas.sh ## clean up RUN rm -rf /var/lib/apt/lists/* /tempdir -RUN conda clean --all -y -f && \ - conda config --add channels conda-forge/label/dev && \ - conda config --add channels conda-forge +RUN conda clean --all -f ## prepare container From 667b4cdf1e03a4980b4ab0b1e5f9962ddd1fdaac Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 08:04:03 -0800 Subject: [PATCH 097/156] pull from later miniconda3 --- worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/Dockerfile b/worker/Dockerfile index 94ff25e..bd24791 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get install -yq --no-install-recommends wget ca-certificates ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PATH /opt/conda/bin:$PATH -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh -O ~/miniconda.sh && \ +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ From 5823f4a0ab01cdfe4b1b63c6a353296aedebb282 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 08:56:39 -0800 Subject: [PATCH 098/156] fix arg for octave worker image --- octave-worker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile index 6d1a8eb..e7f94fe 100644 --- a/octave-worker/Dockerfile +++ b/octave-worker/Dockerfile @@ -1,4 +1,5 @@ -FROM rhodium/worker:$TRAVIS_COMMIT +ARG TRAVIS_COMMIT=${TRAVIS_COMMIT} +FROM rhodium/worker:${TRAVIS_COMMIT} ## using same container as jupyter/base-notebook:python-3.7.3 FROM ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c From 2135501eb7ad3aa85f8a753f91157303af479b47 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 08:56:49 -0800 Subject: [PATCH 099/156] test w/o notebook octave --- notebook/notebook_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 8ba8f0c..a5298e4 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -12,7 +12,7 @@ dependencies: - nbconvert - nodejs - nose - - octave + # - octave - octave_kernel - oct2py - openssh From de49ddc0c1d39e4ee45bf4217e5d703af51ac6b7 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 09:18:52 -0800 Subject: [PATCH 100/156] fix double source image --- octave-worker/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile index e7f94fe..fcafbbe 100644 --- a/octave-worker/Dockerfile +++ b/octave-worker/Dockerfile @@ -1,8 +1,5 @@ ARG TRAVIS_COMMIT=${TRAVIS_COMMIT} FROM rhodium/worker:${TRAVIS_COMMIT} - -## using same container as jupyter/base-notebook:python-3.7.3 -FROM ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c ARG DEBIAN_FRONTEND=noninteractive ## filepath curation From 66cf0167626f695aeb31e5577e927897e2ceb5ed Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 09:29:32 -0800 Subject: [PATCH 101/156] try no octave stuff at all --- notebook/notebook_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index a5298e4..0189056 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -13,8 +13,8 @@ dependencies: - nodejs - nose # - octave - - octave_kernel - - oct2py + # - octave_kernel + # - oct2py - openssh - pytest - pytest-cov From 9d20de7227ed770f426c324c4f317fdf916af2ab Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 12:04:14 -0800 Subject: [PATCH 102/156] move more apt-get into common --- common.sh | 6 ++++-- worker/Dockerfile | 37 ++++++++++++++++++------------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/common.sh b/common.sh index e0aedde..201ce8d 100644 --- a/common.sh +++ b/common.sh @@ -1,14 +1,16 @@ #!/bin/sh # install apt-get packages -apt-get update -y +apt-get update -y --no-install-recommends apt-get install -yq --no-install-recommends \ apt-utils \ bzip2 \ + ca-certificates \ curl \ lsb-release \ gnupg2 \ sudo \ - libgl1-mesa-glx + libgl1-mesa-glx \ + wget # install gcsfuse, google cloud sdk, kubectl # (need curl to be installed earlier) diff --git a/worker/Dockerfile b/worker/Dockerfile index bd24791..1e528e5 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -3,11 +3,26 @@ FROM ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adf ARG DEBIAN_FRONTEND=noninteractive -########### -RUN apt-get update --no-install-recommends -y -RUN apt-get install -yq --no-install-recommends wget ca-certificates +## filepath curation +RUN mkdir /opt/conda/specs +COPY base_environment.yml /opt/conda/specs +RUN mkdir /tempdir +COPY common.sh /tempdir +COPY patch_geopandas.sh /tempdir + +COPY add_service_creds.py /usr/bin +COPY prepare.sh /usr/bin + + +## perform a bunch of common actions +RUN bash /tempdir/common.sh + + +########### +## install miniconda # adapted from continuumio/miniconda3 + ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PATH /opt/conda/bin:$PATH @@ -25,22 +40,6 @@ RUN conda config --add channels conda-forge/label/dev && \ conda config --add channels conda-forge -## filepath curation -RUN mkdir /opt/conda/specs -COPY base_environment.yml /opt/conda/specs - -RUN mkdir /tempdir -COPY common.sh /tempdir -COPY patch_geopandas.sh /tempdir - -COPY add_service_creds.py /usr/bin -COPY prepare.sh /usr/bin - - -## perform a bunch of common actions -RUN bash /tempdir/common.sh - - ## set up python env RUN conda update -n base conda RUN conda env update -f /opt/conda/specs/base_environment.yml From d83b7c94d156d3bc7b0aeb3ccdd950019e574bdd Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 12:24:44 -0800 Subject: [PATCH 103/156] pin python38 --- base_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index a43ce92..35a88ff 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -59,7 +59,7 @@ dependencies: - pygeos - pyinterp - pyshp - - python + - python=3.8 - python-blosc - python-snappy - pyviz_comms From 8a97885555727e3beb62ececcf3767f5aabe59ef Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 12:25:01 -0800 Subject: [PATCH 104/156] remove unnecessary deps --- notebook/notebook_environment.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 0189056..ac2986a 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -7,19 +7,14 @@ dependencies: - flake8 - graphviz - ipdb - - ipywidgets - nano - - nbconvert - - nodejs - nose - # - octave - # - octave_kernel - # - oct2py + - octave + - octave_kernel + - oct2py - openssh - pytest - pytest-cov - pytest-runner - sphinx - tox - - tqdm - - wheel From f06ac03cf01163988bddc193888709615114ba4d Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 12:28:46 -0800 Subject: [PATCH 105/156] remove ruler for notebook b/c slippy --- notebook/overrides.json | 1 - 1 file changed, 1 deletion(-) diff --git a/notebook/overrides.json b/notebook/overrides.json index 64c813f..af86070 100644 --- a/notebook/overrides.json +++ b/notebook/overrides.json @@ -7,7 +7,6 @@ }, "@jupyterlab/notebook-extension:tracker": { "codeCellConfig": { - "rulers": [79], "lineNumbers":true } } From d8bd4e05420c9aa450cb98983e2981ecc1be8f9f Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 12:30:34 -0800 Subject: [PATCH 106/156] fix /opt/conda/specs ref --- notebook/Dockerfile | 10 ++++------ worker/Dockerfile | 5 ++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 78700b3..0605bc1 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -24,12 +24,6 @@ COPY common.sh /tempdir COPY patch_geopandas.sh /tempdir COPY prepare.sh /usr/bin - -RUN mkdir /opt/conda/specs -COPY base_environment.yml /opt/conda/specs -COPY notebook_environment.yml /opt/conda/specs -COPY r_environment.yml /opt/conda/specs - COPY overrides.json /opt/conda/share/jupyter/lab/settings @@ -43,6 +37,10 @@ RUN sudo apt-get install -yq --no-install-recommends vim && sudo apt-get clean # set up conda channels +RUN mkdir /opt/conda/specs +COPY base_environment.yml /opt/conda/specs +COPY notebook_environment.yml /opt/conda/specs +COPY r_environment.yml /opt/conda/specs RUN conda config --add channels conda-forge/label/dev && \ conda config --add channels conda-forge diff --git a/worker/Dockerfile b/worker/Dockerfile index 1e528e5..9fa4429 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -4,9 +4,6 @@ ARG DEBIAN_FRONTEND=noninteractive ## filepath curation -RUN mkdir /opt/conda/specs -COPY base_environment.yml /opt/conda/specs - RUN mkdir /tempdir COPY common.sh /tempdir COPY patch_geopandas.sh /tempdir @@ -36,6 +33,8 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x ## set up conda channels +RUN mkdir /opt/conda/specs +COPY base_environment.yml /opt/conda/specs RUN conda config --add channels conda-forge/label/dev && \ conda config --add channels conda-forge From bcf598413aab436f0de58e3b5328ee63426554c4 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 12:50:33 -0800 Subject: [PATCH 107/156] dont try to bump python --- base_environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index 35a88ff..f5c125a 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -59,7 +59,6 @@ dependencies: - pygeos - pyinterp - pyshp - - python=3.8 - python-blosc - python-snappy - pyviz_comms From cae6ff6183f6c342f081e77406306eec1dae3faf Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 13:20:57 -0800 Subject: [PATCH 108/156] include python in env.yml --- base_environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/base_environment.yml b/base_environment.yml index f5c125a..a43ce92 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -59,6 +59,7 @@ dependencies: - pygeos - pyinterp - pyshp + - python - python-blosc - python-snappy - pyviz_comms From 59e417d5f52809f9b493effcb6fc774901338590 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 14:13:35 -0800 Subject: [PATCH 109/156] fix octave-worker and cleanup travis yml --- .travis.yml | 55 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 11e0909..6a05d42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,12 +8,17 @@ env: - IMAGE_NAME=worker install: -- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak 's/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' notebook/worker-template.yml; else sed -i.bak 's/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' notebook/worker-template.yml; fi" +- "if [[ \"$TRAVIS_TAG\" == \"\" ]]; then sed -i.bak \ +'s/image: rhodium\\/worker.*/image: rhodium\\/worker:'\"$TRAVIS_COMMIT\"'/' \ +notebook/worker-template.yml; else sed -i.bak \ +'s/image: rhodium\\/worker:.*/image: rhodium\\/worker:'\"$TRAVIS_TAG\"'/' \ +notebook/worker-template.yml; fi" - "rm notebook/worker-template.yml.bak" - "cat notebook/worker-template.yml | grep image:" - "cp base_environment.yml $IMAGE_NAME/base_environment.yml" - "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh" -- "cp patch_geopandas.sh $IMAGE_NAME/patch_geopandas.sh && chmod +x $IMAGE_NAME/patch_geopandas.sh" +- "cp patch_geopandas.sh $IMAGE_NAME/patch_geopandas.sh && chmod +x \ +$IMAGE_NAME/patch_geopandas.sh" - "cd $IMAGE_NAME" @@ -22,27 +27,49 @@ script: deploy: - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:$TRAVIS_BRANCH && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" + script: > + docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT + rhodium/$IMAGE_NAME:$TRAVIS_BRANCH && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && + docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" skip_cleanup: true on: all_branches: true condition: $TRAVIS_BRANCH =~ ^dev - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:dev" && docker push "rhodium/$IMAGE_NAME:latest" + script: > + docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && + docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && + docker push "rhodium/$IMAGE_NAME:dev" && + docker push "rhodium/$IMAGE_NAME:latest" skip_cleanup: true on: branch: master - provider: script - script: docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" + script: > + docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT + rhodium/$IMAGE_NAME:$TRAVIS_TAG && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" skip_cleanup: true on: tags: true # octave-worker builds - provider: script - script: docker build -t rhodium/octave-worker:$TRAVIS_COMMIT --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT ../octave-worker && docker tag rhodium/octave-worker:$TRAVIS_COMMIT rhodium/octave-worker:$TRAVIS_BRANCH && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" + script: > + docker build -t rhodium/octave-worker:$TRAVIS_COMMIT + --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT ../octave-worker && + docker tag rhodium/octave-worker:$TRAVIS_COMMIT + rhodium/octave-worker:$TRAVIS_BRANCH && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && + docker push "rhodium/octave-worker:$TRAVIS_BRANCH" skip_cleanup: true on: all_branches: true @@ -50,14 +77,26 @@ deploy: condition: $IMAGE_NAME = worker - provider: script - script: docker tag rhodium/octave-worker:$TRAVIS_COMMIT rhodium/octave-worker:dev && docker tag rhodium/octave-worker$TRAVIS_COMMIT rhodium/octave-worker:latest && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && docker push "rhodium/octave-worker:dev" && docker push "rhodium/octave-worker:latest" + script: > + docker tag rhodium/octave-worker:$TRAVIS_COMMIT + rhodium/octave-worker:dev && + docker tag rhodium/octave-worker$TRAVIS_COMMIT + rhodium/octave-worker:latest && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && + docker push "rhodium/octave-worker:dev" && + docker push "rhodium/octave-worker:latest" skip_cleanup: true on: branch: master condition: $IMAGE_NAME = worker - provider: script - script: docker tag rhodium/octave-worker:$TRAVIS_COMMIT rhodium/octave-worker:$TRAVIS_TAG && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && docker push "rhodium/octave-worker:$TRAVIS_TAG" + script: > + docker tag rhodium/octave-worker:$TRAVIS_COMMIT + rhodium/octave-worker:$TRAVIS_TAG && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/octave-worker:$TRAVIS_TAG" skip_cleanup: true on: tags: true From d13307710c768fcaf556e24c5dcc15c6ecac9f3e Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 14:40:44 -0800 Subject: [PATCH 110/156] try new travis format --- .travis.yml | 80 ++++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a05d42..f82b2bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,49 +27,49 @@ script: deploy: - provider: script - script: > - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT - rhodium/$IMAGE_NAME:$TRAVIS_BRANCH && - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && - docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && - docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" + script: + - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT + rhodium/$IMAGE_NAME:$TRAVIS_BRANCH + - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" + - docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" + - docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" skip_cleanup: true on: all_branches: true condition: $TRAVIS_BRANCH =~ ^dev - provider: script - script: > - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && - docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && - docker push "rhodium/$IMAGE_NAME:dev" && - docker push "rhodium/$IMAGE_NAME:latest" + script: + - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev + - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest + - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" + - docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" + - docker push "rhodium/$IMAGE_NAME:dev" + - docker push "rhodium/$IMAGE_NAME:latest" skip_cleanup: true on: branch: master - provider: script - script: > - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT - rhodium/$IMAGE_NAME:$TRAVIS_TAG && - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && - docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" + script: + - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT + rhodium/$IMAGE_NAME:$TRAVIS_TAG + - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" + - docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" skip_cleanup: true on: tags: true # octave-worker builds - provider: script - script: > - docker build -t rhodium/octave-worker:$TRAVIS_COMMIT - --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT ../octave-worker && - docker tag rhodium/octave-worker:$TRAVIS_COMMIT - rhodium/octave-worker:$TRAVIS_BRANCH && - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && - docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && - docker push "rhodium/octave-worker:$TRAVIS_BRANCH" + script: + - docker build -t rhodium/octave-worker:$TRAVIS_COMMIT + --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT ../octave-worker + - docker tag rhodium/octave-worker:$TRAVIS_COMMIT + rhodium/octave-worker:$TRAVIS_BRANCH + - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" + - docker push "rhodium/octave-worker:$TRAVIS_COMMIT" + - docker push "rhodium/octave-worker:$TRAVIS_BRANCH" skip_cleanup: true on: all_branches: true @@ -77,26 +77,26 @@ deploy: condition: $IMAGE_NAME = worker - provider: script - script: > - docker tag rhodium/octave-worker:$TRAVIS_COMMIT - rhodium/octave-worker:dev && - docker tag rhodium/octave-worker$TRAVIS_COMMIT - rhodium/octave-worker:latest && - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && - docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && - docker push "rhodium/octave-worker:dev" && - docker push "rhodium/octave-worker:latest" + script: + - docker tag rhodium/octave-worker:$TRAVIS_COMMIT + rhodium/octave-worker:dev + - docker tag rhodium/octave-worker$TRAVIS_COMMIT + rhodium/octave-worker:latest + - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" + - docker push "rhodium/octave-worker:$TRAVIS_COMMIT" + - docker push "rhodium/octave-worker:dev" + - docker push "rhodium/octave-worker:latest" skip_cleanup: true on: branch: master condition: $IMAGE_NAME = worker - provider: script - script: > - docker tag rhodium/octave-worker:$TRAVIS_COMMIT - rhodium/octave-worker:$TRAVIS_TAG && - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && - docker push "rhodium/octave-worker:$TRAVIS_TAG" + script: + - docker tag rhodium/octave-worker:$TRAVIS_COMMIT + rhodium/octave-worker:$TRAVIS_TAG + - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" + - docker push "rhodium/octave-worker:$TRAVIS_TAG" skip_cleanup: true on: tags: true From 29cfc67e12c8d921cd1d8931308a933e91c929a7 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 15:02:11 -0800 Subject: [PATCH 111/156] try new travis again --- .travis.yml | 80 ++++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.travis.yml b/.travis.yml index f82b2bf..e6c6353 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,49 +27,49 @@ script: deploy: - provider: script - script: - - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT - rhodium/$IMAGE_NAME:$TRAVIS_BRANCH - - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" - - docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" - - docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" + script: >- + docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT + rhodium/$IMAGE_NAME:$TRAVIS_BRANCH && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && + docker push "rhodium/$IMAGE_NAME:$TRAVIS_BRANCH" skip_cleanup: true on: all_branches: true condition: $TRAVIS_BRANCH =~ ^dev - provider: script - script: - - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev - - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest - - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" - - docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" - - docker push "rhodium/$IMAGE_NAME:dev" - - docker push "rhodium/$IMAGE_NAME:latest" + script: >- + docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:dev && + docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT rhodium/$IMAGE_NAME:latest && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/$IMAGE_NAME:$TRAVIS_COMMIT" && + docker push "rhodium/$IMAGE_NAME:dev" && + docker push "rhodium/$IMAGE_NAME:latest" skip_cleanup: true on: branch: master - provider: script - script: - - docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT - rhodium/$IMAGE_NAME:$TRAVIS_TAG - - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" - - docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" + script: >- + docker tag rhodium/$IMAGE_NAME:$TRAVIS_COMMIT + rhodium/$IMAGE_NAME:$TRAVIS_TAG && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/$IMAGE_NAME:$TRAVIS_TAG" skip_cleanup: true on: tags: true # octave-worker builds - provider: script - script: - - docker build -t rhodium/octave-worker:$TRAVIS_COMMIT - --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT ../octave-worker - - docker tag rhodium/octave-worker:$TRAVIS_COMMIT - rhodium/octave-worker:$TRAVIS_BRANCH - - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" - - docker push "rhodium/octave-worker:$TRAVIS_COMMIT" - - docker push "rhodium/octave-worker:$TRAVIS_BRANCH" + script: >- + docker build -t rhodium/octave-worker:$TRAVIS_COMMIT + --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT ../octave-worker && + docker tag rhodium/octave-worker:$TRAVIS_COMMIT + rhodium/octave-worker:$TRAVIS_BRANCH && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && + docker push "rhodium/octave-worker:$TRAVIS_BRANCH" skip_cleanup: true on: all_branches: true @@ -77,26 +77,26 @@ deploy: condition: $IMAGE_NAME = worker - provider: script - script: - - docker tag rhodium/octave-worker:$TRAVIS_COMMIT - rhodium/octave-worker:dev - - docker tag rhodium/octave-worker$TRAVIS_COMMIT - rhodium/octave-worker:latest - - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" - - docker push "rhodium/octave-worker:$TRAVIS_COMMIT" - - docker push "rhodium/octave-worker:dev" - - docker push "rhodium/octave-worker:latest" + script: >- + docker tag rhodium/octave-worker:$TRAVIS_COMMIT + rhodium/octave-worker:dev && + docker tag rhodium/octave-worker$TRAVIS_COMMIT + rhodium/octave-worker:latest && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/octave-worker:$TRAVIS_COMMIT" && + docker push "rhodium/octave-worker:dev" && + docker push "rhodium/octave-worker:latest" skip_cleanup: true on: branch: master condition: $IMAGE_NAME = worker - provider: script - script: - - docker tag rhodium/octave-worker:$TRAVIS_COMMIT - rhodium/octave-worker:$TRAVIS_TAG - - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" - - docker push "rhodium/octave-worker:$TRAVIS_TAG" + script: >- + docker tag rhodium/octave-worker:$TRAVIS_COMMIT + rhodium/octave-worker:$TRAVIS_TAG && + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && + docker push "rhodium/octave-worker:$TRAVIS_TAG" skip_cleanup: true on: tags: true From 96e0be6e761ca591f2bd2942cabe115d48a2df91 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sun, 17 Nov 2019 15:38:47 -0800 Subject: [PATCH 112/156] pin a few pacakges --- base_environment.yml | 7 +++---- notebook/notebook_environment.yml | 1 + octave-worker/octave_environment.yml | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index a43ce92..97c925a 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -21,10 +21,10 @@ dependencies: - dropbox - esmpy - fastparquet - - fiona + - fiona=1.8.9.post2 - fusepy - gcsfs - - gdal + - gdal=2.4.3 - geoalchemy2 - geopandas=1.0.0.dev - geopy @@ -32,7 +32,6 @@ dependencies: - geoviews - git - gitpython - - gnuplot - google-cloud-container - google-cloud-storage - holoviews @@ -64,7 +63,7 @@ dependencies: - python-snappy - pyviz_comms - PyYAML - - rasterio + - rasterio=1.0.28 - regionmask - scikit-image - scikit-learn diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index ac2986a..ade9071 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -5,6 +5,7 @@ dependencies: - coverage - dask-labextension - flake8 + - gnuplot=5.2.7 - graphviz - ipdb - nano diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml index 79bdc91..13cf6d5 100644 --- a/octave-worker/octave_environment.yml +++ b/octave-worker/octave_environment.yml @@ -2,6 +2,7 @@ name: base channels: - conda-forge dependencies: + - gnuplot=5.2.7 - octave - octave_kernel - oct2py From 13a6b0c441479fe962b075b3c0b699f7ec357e3f Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 18 Nov 2019 00:04:37 -0800 Subject: [PATCH 113/156] install octave from apt --- notebook/Dockerfile | 5 ++++- notebook/notebook_environment.yml | 1 - octave-worker/Dockerfile | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 0605bc1..f13ceea 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -33,7 +33,10 @@ RUN sudo chown -R $NB_USER /gcs ## more apt-get -RUN sudo apt-get install -yq --no-install-recommends vim && sudo apt-get clean +RUN sudo apt-get install -yq --no-install-recommends + vim \ + octave && \ + sudo apt-get clean # set up conda channels diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index ade9071..3cf360a 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -10,7 +10,6 @@ dependencies: - ipdb - nano - nose - - octave - octave_kernel - oct2py - openssh diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile index fcafbbe..19bb929 100644 --- a/octave-worker/Dockerfile +++ b/octave-worker/Dockerfile @@ -5,5 +5,8 @@ ARG DEBIAN_FRONTEND=noninteractive ## filepath curation COPY octave_environment.yml /opt/conda/specs/octave_environment.yml +## install octave +RUN sudo apt-get install --no-install-recommends octave + # add octave-specific packages RUN conda env update -f /opt/conda/specs/octave_environment.yml From 0721b5138603986f113256a5046889c899a89f35 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 18 Nov 2019 00:10:50 -0800 Subject: [PATCH 114/156] fix typo --- notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index f13ceea..676b40b 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -33,7 +33,7 @@ RUN sudo chown -R $NB_USER /gcs ## more apt-get -RUN sudo apt-get install -yq --no-install-recommends +RUN sudo apt-get install -yq --no-install-recommends \ vim \ octave && \ sudo apt-get clean From 3118cfdf3777402e724b8add227b0c6c36ba176d Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 18 Nov 2019 06:52:46 -0800 Subject: [PATCH 115/156] update apt gets on octave-worker --- octave-worker/Dockerfile | 3 ++- octave-worker/octave_environment.yml | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile index 19bb929..ea50b4a 100644 --- a/octave-worker/Dockerfile +++ b/octave-worker/Dockerfile @@ -6,7 +6,8 @@ ARG DEBIAN_FRONTEND=noninteractive COPY octave_environment.yml /opt/conda/specs/octave_environment.yml ## install octave -RUN sudo apt-get install --no-install-recommends octave +RUN sudo apt-get update --no-install-recommends && \ + sudo apt-get install -q --no-install-recommends octave # add octave-specific packages RUN conda env update -f /opt/conda/specs/octave_environment.yml diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml index 13cf6d5..702c886 100644 --- a/octave-worker/octave_environment.yml +++ b/octave-worker/octave_environment.yml @@ -3,6 +3,5 @@ channels: - conda-forge dependencies: - gnuplot=5.2.7 - - octave - octave_kernel - oct2py From 3b02164fb005b284f3edcef8d2d675eada290894 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 18 Nov 2019 07:17:53 -0800 Subject: [PATCH 116/156] yq the apt get --- octave-worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile index ea50b4a..b13f6c3 100644 --- a/octave-worker/Dockerfile +++ b/octave-worker/Dockerfile @@ -7,7 +7,7 @@ COPY octave_environment.yml /opt/conda/specs/octave_environment.yml ## install octave RUN sudo apt-get update --no-install-recommends && \ - sudo apt-get install -q --no-install-recommends octave + sudo apt-get install -yq --no-install-recommends octave # add octave-specific packages RUN conda env update -f /opt/conda/specs/octave_environment.yml From 0b4701bf3f385989f0c7880c8e012a8873076866 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 18 Nov 2019 10:56:35 -0800 Subject: [PATCH 117/156] add ghostscript --- notebook/notebook_environment.yml | 1 + octave-worker/octave_environment.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 3cf360a..218a35f 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -5,6 +5,7 @@ dependencies: - coverage - dask-labextension - flake8 + - ghostscript - gnuplot=5.2.7 - graphviz - ipdb diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml index 702c886..a6e07af 100644 --- a/octave-worker/octave_environment.yml +++ b/octave-worker/octave_environment.yml @@ -2,6 +2,7 @@ name: base channels: - conda-forge dependencies: + - ghostscript - gnuplot=5.2.7 - octave_kernel - oct2py From 23806e1caca6501bd3a89aea0f71615172bc4cb3 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 18 Nov 2019 11:11:47 -0800 Subject: [PATCH 118/156] fix rulers --- notebook/Dockerfile | 2 +- notebook/overrides.json | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 676b40b..173f412 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -24,7 +24,7 @@ COPY common.sh /tempdir COPY patch_geopandas.sh /tempdir COPY prepare.sh /usr/bin -COPY overrides.json /opt/conda/share/jupyter/lab/settings +COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json ## perform a bunch of common actions diff --git a/notebook/overrides.json b/notebook/overrides.json index af86070..82b1316 100644 --- a/notebook/overrides.json +++ b/notebook/overrides.json @@ -1,13 +1,7 @@ { "@jupyterlab/fileeditor-extension:plugin": { "editorConfig": { - "rulers": [79], - "lineNumbers":true - } - }, - "@jupyterlab/notebook-extension:tracker": { - "codeCellConfig": { - "lineNumbers":true + "rulers": [79] } } } From 49aba4b280cc2d0edc252756f8a4bce5a04ea57b Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 18 Nov 2019 13:14:09 -0800 Subject: [PATCH 119/156] add line numbers back --- notebook/overrides.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/notebook/overrides.json b/notebook/overrides.json index 82b1316..78dcf65 100644 --- a/notebook/overrides.json +++ b/notebook/overrides.json @@ -1,7 +1,8 @@ { "@jupyterlab/fileeditor-extension:plugin": { "editorConfig": { - "rulers": [79] + "rulers": [79], + "lineNumbers": true } } } From 0f43dee9b1ea34cfb7c0a690fac1599176efd08b Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 09:30:10 -0800 Subject: [PATCH 120/156] switch to prod geopandas and fix gcc for clawpack --- base_environment.yml | 17 ++++++++++++----- notebook/Dockerfile | 4 ---- notebook/notebook_environment.yml | 4 +++- octave-worker/octave_environment.yml | 3 ++- patch_geopandas.sh | 12 ------------ worker/Dockerfile | 5 ----- 6 files changed, 17 insertions(+), 28 deletions(-) delete mode 100644 patch_geopandas.sh diff --git a/base_environment.yml b/base_environment.yml index 97c925a..259fe22 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -1,7 +1,6 @@ name: base channels: - conda-forge - - conda-forge/label/dev dependencies: - beautifulsoup4 - boto3 @@ -21,12 +20,19 @@ dependencies: - dropbox - esmpy - fastparquet - - fiona=1.8.9.post2 + - fiona + # - fiona=1.8.9.post2 - fusepy +# this gcc pin is necessary b/c of a weird feature in the h553295d_15 build +# which makes it hard to build numpy-based cython extensions (like pyclaw). +# we should try removing it whenever we next do an update and see if Clawpack +# can still be built + - gcc_linux-64=7.3.0=h553295d_14 - gcsfs - - gdal=2.4.3 + - gdal + # - gdal=2.4.3 - geoalchemy2 - - geopandas=1.0.0.dev + - geopandas - geopy - geotiff - geoviews @@ -63,7 +69,8 @@ dependencies: - python-snappy - pyviz_comms - PyYAML - - rasterio=1.0.28 + - rasterio + # - rasterio=1.0.28 - regionmask - scikit-image - scikit-learn diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 173f412..08cf799 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -80,10 +80,6 @@ RUN jupyter serverextension enable --sys-prefix jupyter_server_proxy RUN jupyter serverextension enable --py --sys-prefix dask_labextension -## needed until cythonized geopandas updated for new pandas -RUN /tempdir/patch_geopandas.sh - - ## clean up RUN sudo rm -rf /var/lib/apt/lists/* /tempdir RUN conda clean --all -f diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 218a35f..edf62ed 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -6,10 +6,12 @@ dependencies: - dask-labextension - flake8 - ghostscript - - gnuplot=5.2.7 + - gnuplot + # - gnuplot=5.2.7 - graphviz - ipdb - nano + - nb_conda_kernels - nose - octave_kernel - oct2py diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml index a6e07af..3691279 100644 --- a/octave-worker/octave_environment.yml +++ b/octave-worker/octave_environment.yml @@ -3,6 +3,7 @@ channels: - conda-forge dependencies: - ghostscript - - gnuplot=5.2.7 + - gnuplot + # - gnuplot=5.2.7 - octave_kernel - oct2py diff --git a/patch_geopandas.sh b/patch_geopandas.sh deleted file mode 100644 index 05ce4cc..0000000 --- a/patch_geopandas.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# manually edit geopandas/label/dev to match current pandas deployment -# see https://github.com/geopandas/geopandas/issues/473#issuecomment-465060849 -# should be removed if geopandas core becomes vectorized, whether the -# cythonized branch gets merged in or the geometry calculations are handled -# by pyGEOS, either through shapely or replacing shapely. See -# https://github.com/Toblerity/Shapely/issues/782. -sed -i "s/from pandas.core.internals import Block, NonConsolidatableMixIn, \ -BlockManager/from pandas.core.internals.blocks import Block, \ -NonConsolidatableMixIn\nfrom pandas.core.internals.managers import \ -BlockManager/g" /opt/conda/lib/python3.6/site-packages/geopandas/_block.py diff --git a/worker/Dockerfile b/worker/Dockerfile index 9fa4429..d5e000c 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -6,7 +6,6 @@ ARG DEBIAN_FRONTEND=noninteractive ## filepath curation RUN mkdir /tempdir COPY common.sh /tempdir -COPY patch_geopandas.sh /tempdir COPY add_service_creds.py /usr/bin COPY prepare.sh /usr/bin @@ -44,10 +43,6 @@ RUN conda update -n base conda RUN conda env update -f /opt/conda/specs/base_environment.yml -## needed until cythonized geopandas updated for new pandas -RUN /tempdir/patch_geopandas.sh - - ## clean up RUN rm -rf /var/lib/apt/lists/* /tempdir RUN conda clean --all -f From b3f1f28cbaa1b178d97626d0b7e14c8e76dfc620 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 09:32:34 -0800 Subject: [PATCH 121/156] fix patch geopandas --- .travis.yml | 2 -- notebook/Dockerfile | 1 - 2 files changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6c6353..f62ddaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,6 @@ notebook/worker-template.yml; fi" - "cat notebook/worker-template.yml | grep image:" - "cp base_environment.yml $IMAGE_NAME/base_environment.yml" - "cp common.sh $IMAGE_NAME/common.sh && chmod +x $IMAGE_NAME/common.sh" -- "cp patch_geopandas.sh $IMAGE_NAME/patch_geopandas.sh && chmod +x \ -$IMAGE_NAME/patch_geopandas.sh" - "cd $IMAGE_NAME" diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 08cf799..c74b630 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -21,7 +21,6 @@ COPY config.yaml /pre-home RUN sudo mkdir /tempdir COPY common.sh /tempdir -COPY patch_geopandas.sh /tempdir COPY prepare.sh /usr/bin COPY overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json From 8b470079e1dd5d9487fe63c90aa3e626095de9ea Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 09:48:00 -0800 Subject: [PATCH 122/156] remove nb_conda_kernels from base --- base_environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index 259fe22..e293d6f 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -50,7 +50,6 @@ dependencies: - lz4 - make - matplotlib - - nb_conda_kernels - nc-time-axis - ncurses - netcdf-fortran From 433537427bac3a9453ea5d7cf808e0a192196de4 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 10:16:17 -0800 Subject: [PATCH 123/156] add black and change octave deps --- base_environment.yml | 3 --- notebook/Dockerfile | 4 +++- notebook/notebook_environment.yml | 7 ++++--- notebook/overrides.json | 7 +++++++ octave-worker/Dockerfile | 3 +-- octave-worker/octave_environment.yml | 3 --- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index e293d6f..fe47074 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -21,7 +21,6 @@ dependencies: - esmpy - fastparquet - fiona - # - fiona=1.8.9.post2 - fusepy # this gcc pin is necessary b/c of a weird feature in the h553295d_15 build # which makes it hard to build numpy-based cython extensions (like pyclaw). @@ -30,7 +29,6 @@ dependencies: - gcc_linux-64=7.3.0=h553295d_14 - gcsfs - gdal - # - gdal=2.4.3 - geoalchemy2 - geopandas - geopy @@ -69,7 +67,6 @@ dependencies: - pyviz_comms - PyYAML - rasterio - # - rasterio=1.0.28 - regionmask - scikit-image - scikit-learn diff --git a/notebook/Dockerfile b/notebook/Dockerfile index c74b630..80f792a 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -32,7 +32,7 @@ RUN sudo chown -R $NB_USER /gcs ## more apt-get -RUN sudo apt-get install -yq --no-install-recommends \ +RUN sudo apt-get install -yq \ vim \ octave && \ sudo apt-get clean @@ -68,6 +68,7 @@ RUN jupyter labextension update --all && \ @jupyter-widgets/jupyterlab-manager \ @jupyter-widgets/jupyterlab-sidecar \ @pyviz/jupyterlab_pyviz \ + @ryantam626/jupyterlab_black \ dask-labextension \ itk-jupyter-widgets \ jupyterlab_bokeh \ @@ -77,6 +78,7 @@ RUN jupyter serverextension enable --py jupyterlab --sys-prefix RUN jupyter serverextension enable --py jupyterlab --user RUN jupyter serverextension enable --sys-prefix jupyter_server_proxy RUN jupyter serverextension enable --py --sys-prefix dask_labextension +RUN jupyter serverextension enable --py jupyterlab_black ## clean up diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index edf62ed..b53dcab 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -2,12 +2,10 @@ name: base channels: - conda-forge dependencies: + - black - coverage - dask-labextension - flake8 - - ghostscript - - gnuplot - # - gnuplot=5.2.7 - graphviz - ipdb - nano @@ -21,3 +19,6 @@ dependencies: - pytest-runner - sphinx - tox + - pip: + - black_nbconvert + - https://github.com/ryantam626/jupyterlab_black/archive/master.zip diff --git a/notebook/overrides.json b/notebook/overrides.json index 78dcf65..54d2593 100644 --- a/notebook/overrides.json +++ b/notebook/overrides.json @@ -6,3 +6,10 @@ } } } + +{ + "@ryantam626/jupyterlab_black:settings": { + "blackPythonBin": "/opt/conda/bin/python", + "lineLength": 79 + } +} diff --git a/octave-worker/Dockerfile b/octave-worker/Dockerfile index b13f6c3..75039f0 100644 --- a/octave-worker/Dockerfile +++ b/octave-worker/Dockerfile @@ -6,8 +6,7 @@ ARG DEBIAN_FRONTEND=noninteractive COPY octave_environment.yml /opt/conda/specs/octave_environment.yml ## install octave -RUN sudo apt-get update --no-install-recommends && \ - sudo apt-get install -yq --no-install-recommends octave +RUN sudo apt-get update && sudo apt-get install -yq octave # add octave-specific packages RUN conda env update -f /opt/conda/specs/octave_environment.yml diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml index 3691279..03fc43e 100644 --- a/octave-worker/octave_environment.yml +++ b/octave-worker/octave_environment.yml @@ -2,8 +2,5 @@ name: base channels: - conda-forge dependencies: - - ghostscript - - gnuplot - # - gnuplot=5.2.7 - octave_kernel - oct2py From 0179f398cb24432873f245a0ecefe4c002dca1c4 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 11:22:31 -0800 Subject: [PATCH 124/156] drop notebook black --- notebook/Dockerfile | 16 ++++------------ notebook/notebook_environment.yml | 4 +++- notebook/overrides.json | 7 ------- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 80f792a..85ad22d 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -63,22 +63,14 @@ RUN conda env create -f /opt/conda/specs/r_environment.yml ## Set up jupyter lab extensions RUN jupyter labextension update --all && \ jupyter labextension install \ - @jupyterlab/hub-extension \ - @jupyterlab/plotly-extension \ + @bokeh/jupyter_bokeh \ @jupyter-widgets/jupyterlab-manager \ @jupyter-widgets/jupyterlab-sidecar \ @pyviz/jupyterlab_pyviz \ - @ryantam626/jupyterlab_black \ dask-labextension \ - itk-jupyter-widgets \ - jupyterlab_bokeh \ - jupyter-leaflet - -RUN jupyter serverextension enable --py jupyterlab --sys-prefix -RUN jupyter serverextension enable --py jupyterlab --user -RUN jupyter serverextension enable --sys-prefix jupyter_server_proxy -RUN jupyter serverextension enable --py --sys-prefix dask_labextension -RUN jupyter serverextension enable --py jupyterlab_black + jupyter-leaflet \ + jupyterlab-plotly \ + plotlywidget ## clean up diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index b53dcab..8e03647 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -8,6 +8,9 @@ dependencies: - flake8 - graphviz - ipdb + - ipyleaflet + - jupyter_bokeh + - jupyterlab_code_formatter - nano - nb_conda_kernels - nose @@ -21,4 +24,3 @@ dependencies: - tox - pip: - black_nbconvert - - https://github.com/ryantam626/jupyterlab_black/archive/master.zip diff --git a/notebook/overrides.json b/notebook/overrides.json index 54d2593..78dcf65 100644 --- a/notebook/overrides.json +++ b/notebook/overrides.json @@ -6,10 +6,3 @@ } } } - -{ - "@ryantam626/jupyterlab_black:settings": { - "blackPythonBin": "/opt/conda/bin/python", - "lineLength": 79 - } -} From 57d98def4d705168b50439f19ee0242c60b684a9 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 11:47:39 -0800 Subject: [PATCH 125/156] remove jupyter_bokeh --- notebook/notebook_environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 8e03647..47f8a98 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -6,10 +6,8 @@ dependencies: - coverage - dask-labextension - flake8 - - graphviz - ipdb - ipyleaflet - - jupyter_bokeh - jupyterlab_code_formatter - nano - nb_conda_kernels @@ -20,6 +18,7 @@ dependencies: - pytest - pytest-cov - pytest-runner + - python-graphviz - sphinx - tox - pip: From 31045c1f43bf3768f2130bf3bdcf52bfa427b555 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 12:28:44 -0800 Subject: [PATCH 126/156] reset worker env vars --- worker/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/worker/Dockerfile b/worker/Dockerfile index d5e000c..adb0aee 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -50,5 +50,7 @@ RUN conda clean --all -f ## prepare container ENV OMP_NUM_THREADS=1 -ENV DASK_TICK_MAXIMUM_DELAY=5s +ENV MKL_NUM_THREADS=1 +ENV OPENBLAS_NUM_THREADS=1 + ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] From 4445254415ee43d7a75523bfef92af3715022b13 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 13:43:17 -0800 Subject: [PATCH 127/156] add jupyter-server-proxy for remote scheduler --- base_environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index fe47074..2afffe5 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -41,8 +41,9 @@ dependencies: - holoviews - h5netcdf - icu - - ipyleaflet - jedi +# need server proxy on workers if using remote scheduler + - jupyter-server-proxy - kubernetes - lapack - lz4 From 10c517527b3dec503e3c7aac547836f332ee9906 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 17:07:37 -0800 Subject: [PATCH 128/156] point to updated rhg_compute_tools --- base_environment.yml | 2 +- notebook/config.yaml | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 2afffe5..ac50824 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -95,5 +95,5 @@ dependencies: - climate-toolbox - impactlab-tools - parameterize_jobs - - rhg_compute_tools + - git+https://github.com/rhodiumgroup/rhg_compute_tools.git@allow-remote-scheduler - git+https://github.com/NCAR/intake-esm.git diff --git a/notebook/config.yaml b/notebook/config.yaml index 2b88bc1..61d9075 100644 --- a/notebook/config.yaml +++ b/notebook/config.yaml @@ -2,10 +2,5 @@ distributed: dashboard: link: "{JUPYTERHUB_SERVICE_PREFIX}proxy/{port}/status" - admin: - tick: - limit: 5s - kubernetes: worker-template-path: "/home/{NB_USER}/worker-template.yml" - name: "dask-{JUPYTERHUB_USER}-{uuid}" \ No newline at end of file From 0a0e50109ccea0550509ae7b02db5ba023edb1fe Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Sat, 23 Nov 2019 21:27:33 -0800 Subject: [PATCH 129/156] fix bokeh/dashboard arg --- notebook/worker-template.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/notebook/worker-template.yml b/notebook/worker-template.yml index c1686b6..a8990c2 100644 --- a/notebook/worker-template.yml +++ b/notebook/worker-template.yml @@ -2,8 +2,15 @@ metadata: spec: restartPolicy: Never containers: - - args: [dask-worker, $(DASK_SCHEDULER_ADDRESS), --nthreads, '1', --no-bokeh, - --memory-limit, 11.5GB, --death-timeout, '60'] + - args: + - dask-worker + - --nthreads + - '1' + - --no-dashboard + - --memory-limit + - 11.5GB + - --death-timeout + - '60' env: - name: GCSFUSE_BUCKET value: rhg-data From cedafe4f41454f79f4c7527e2ebdb775840868c1 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 25 Nov 2019 11:34:43 -0800 Subject: [PATCH 130/156] put jupyterhub_user back in config --- notebook/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/notebook/config.yaml b/notebook/config.yaml index 61d9075..2aec4df 100644 --- a/notebook/config.yaml +++ b/notebook/config.yaml @@ -4,3 +4,4 @@ distributed: kubernetes: worker-template-path: "/home/{NB_USER}/worker-template.yml" + name: "dask-{JUPYTERHUB_USER}-{uuid}" From 4444a717c6b4e10796a0a3ba2871f03b2613a890 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 25 Nov 2019 12:30:25 -0800 Subject: [PATCH 131/156] add in other things from pangeo stacks --- notebook/config.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/notebook/config.yaml b/notebook/config.yaml index 2aec4df..c9231d7 100644 --- a/notebook/config.yaml +++ b/notebook/config.yaml @@ -1,7 +1,28 @@ +# this config file is adapted from the one used by Pangeo: +# https://github.com/pangeo-data/pangeo-stacks/blob/master/base-notebook/binder/dask_config.yaml + distributed: dashboard: link: "{JUPYTERHUB_SERVICE_PREFIX}proxy/{port}/status" + admin: + tick: + limit: 5s + +logging: + distributed: warning + bokeh: critical + # http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr + tornado: critical + tornado.application: error + +labextension: + factory: + module: dask_kubernetes + class: KubeCluster + args: [] + kwargs: {} + kubernetes: worker-template-path: "/home/{NB_USER}/worker-template.yml" name: "dask-{JUPYTERHUB_USER}-{uuid}" From da95509f744bd4c98dfd28819e2870d941a3ce97 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 25 Nov 2019 13:47:21 -0800 Subject: [PATCH 132/156] add comments to yaml sections --- notebook/config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/notebook/config.yaml b/notebook/config.yaml index c9231d7..3e8ec1a 100644 --- a/notebook/config.yaml +++ b/notebook/config.yaml @@ -16,6 +16,10 @@ logging: tornado: critical tornado.application: error +# the below section gives defaults for launching clusters from the dask +# labextension viewer. We could customize this but right now it should load +# a default cluster from the worker-template.yml file (as sepecified in the +# following section) labextension: factory: module: dask_kubernetes From cff93c824b6b40e16f144129188b704beee5c172 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Tue, 26 Nov 2019 15:19:17 -0800 Subject: [PATCH 133/156] ipympl extension --- notebook/Dockerfile | 1 + notebook/notebook_environment.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 85ad22d..1b4ed8f 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -69,6 +69,7 @@ RUN jupyter labextension update --all && \ @pyviz/jupyterlab_pyviz \ dask-labextension \ jupyter-leaflet \ + jupyter-matplotlib \ jupyterlab-plotly \ plotlywidget diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 47f8a98..3b3a322 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -8,6 +8,7 @@ dependencies: - flake8 - ipdb - ipyleaflet + - ipympl - jupyterlab_code_formatter - nano - nb_conda_kernels From c9a90c7266d833d9e1775b8b57098e2f9fdf2d4b Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 29 Nov 2019 12:15:55 -0800 Subject: [PATCH 134/156] pull dk from master --- base_environment.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index ac50824..f0aa9ac 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -13,7 +13,6 @@ dependencies: - compilers - dask - dask-glm - - dask-kubernetes - dask-ml - datashader - distributed @@ -97,3 +96,7 @@ dependencies: - parameterize_jobs - git+https://github.com/rhodiumgroup/rhg_compute_tools.git@allow-remote-scheduler - git+https://github.com/NCAR/intake-esm.git +# need to install from master until 0.10.1 +# 6762b474dd61f316bd952db0fa1d3ad96dc54174 is a known good commit but for now +# not pinning the hash + - git+https://github.com/dask/dask-kubernetes From 7f8ca3e6ada73e3ad40b139486e13a79795a1dde Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 29 Nov 2019 17:54:30 -0800 Subject: [PATCH 135/156] put testing pkgs in base env --- base_environment.yml | 5 +++++ notebook/notebook_environment.yml | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index f0aa9ac..9067775 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -61,6 +61,9 @@ dependencies: - pygeos - pyinterp - pyshp + - pytest + - pytest-cov + - pytest-runner - python - python-blosc - python-snappy @@ -74,8 +77,10 @@ dependencies: - seaborn - shapely - sparse + - sphinx - statsmodels - tini + - tox - unzip - uritemplate - xarray diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 3b3a322..0607170 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -16,11 +16,7 @@ dependencies: - octave_kernel - oct2py - openssh - - pytest - - pytest-cov - - pytest-runner - python-graphviz - - sphinx - tox - pip: - black_nbconvert From 3a0339ca8daa68a2cbbc8ecb848b835d5f5c7a8c Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 29 Nov 2019 17:56:41 -0800 Subject: [PATCH 136/156] nope dont want that --- base_environment.yml | 5 ----- notebook/notebook_environment.yml | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 9067775..f0aa9ac 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -61,9 +61,6 @@ dependencies: - pygeos - pyinterp - pyshp - - pytest - - pytest-cov - - pytest-runner - python - python-blosc - python-snappy @@ -77,10 +74,8 @@ dependencies: - seaborn - shapely - sparse - - sphinx - statsmodels - tini - - tox - unzip - uritemplate - xarray diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 0607170..3b3a322 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -16,7 +16,11 @@ dependencies: - octave_kernel - oct2py - openssh + - pytest + - pytest-cov + - pytest-runner - python-graphviz + - sphinx - tox - pip: - black_nbconvert From c189c49077ca5ffe36c9e538b7fcd89f39eedc66 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 2 Dec 2019 14:23:40 -0800 Subject: [PATCH 137/156] set numpy install flag --- notebook/Dockerfile | 4 ++++ worker/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 1b4ed8f..11c21b6 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,6 +1,10 @@ FROM jupyter/base-notebook:latest ARG DEBIAN_FRONTEND=noninteractive +## needed to make sure things with cython compile correctly +## (this will eventually become default in numpy) +ENV NPY_DISTUTILS_APPEND_FLAGS=1 + ## get rid of conda pin file that causes update problems RUN rm /opt/conda/conda-meta/pinned diff --git a/worker/Dockerfile b/worker/Dockerfile index adb0aee..bde5438 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -1,7 +1,7 @@ ## using same container as jupyter/base-notebook:python-3.7.3 FROM ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c ARG DEBIAN_FRONTEND=noninteractive - +ENV NPY_DISTUTILS_APPEND_FLAGS=1 ## filepath curation RUN mkdir /tempdir From d3ba18c7d00eaa58550a8959f46cc024a3c1995a Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Mon, 2 Dec 2019 21:39:50 -0800 Subject: [PATCH 138/156] pin rtree --- base_environment.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base_environment.yml b/base_environment.yml index f0aa9ac..04eb96d 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -68,6 +68,9 @@ dependencies: - PyYAML - rasterio - regionmask +# the pinned rtree can be deleted from this requirements file once this is +# solved and a new release is made: https://github.com/Toblerity/rtree/issues/124 + - rtree=0.8.3 - scikit-image - scikit-learn - scipy From 3adbb88182f866d6d8feb7b6f7ec04fe3f81470e Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Tue, 3 Dec 2019 09:34:53 -0800 Subject: [PATCH 139/156] pin esmpy --- base_environment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index 04eb96d..c21471a 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -17,7 +17,9 @@ dependencies: - datashader - distributed - dropbox - - esmpy +# need to make sure we get esmpy compiled with mpi otherwise xesmf regridding +# won't work + - esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101 - fastparquet - fiona - fusepy From 52a7a33268a0c776fba3e99ff45574fda976e3f1 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Tue, 3 Dec 2019 11:48:52 -0800 Subject: [PATCH 140/156] try forcing dask-kubernetes to install master --- base_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index c21471a..918f042 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -104,4 +104,4 @@ dependencies: # need to install from master until 0.10.1 # 6762b474dd61f316bd952db0fa1d3ad96dc54174 is a known good commit but for now # not pinning the hash - - git+https://github.com/dask/dask-kubernetes + - git+https://github.com/dask/dask-kubernetes@master From 085a1846e10e8f7f7488b49d202da3513eb22401 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Tue, 3 Dec 2019 12:32:14 -0800 Subject: [PATCH 141/156] pin dk commit --- base_environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 918f042..70cf495 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -102,6 +102,6 @@ dependencies: - git+https://github.com/rhodiumgroup/rhg_compute_tools.git@allow-remote-scheduler - git+https://github.com/NCAR/intake-esm.git # need to install from master until 0.10.1 -# 6762b474dd61f316bd952db0fa1d3ad96dc54174 is a known good commit but for now -# not pinning the hash - - git+https://github.com/dask/dask-kubernetes@master +# due to handling of remote scheduler +# (we also should at some point switch to dask-gateway instead of dask-kubernetes) + - git+https://github.com/dask/dask-kubernetes@b481bbbc6c1bb7d0719bc385c0537c4ba4a0ee57 From 748fde8e4e6c0ad6705c5c2ebb029aed888882b6 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Tue, 3 Dec 2019 15:08:17 -0800 Subject: [PATCH 142/156] retry dk --- base_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_environment.yml b/base_environment.yml index 70cf495..bf24671 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -104,4 +104,4 @@ dependencies: # need to install from master until 0.10.1 # due to handling of remote scheduler # (we also should at some point switch to dask-gateway instead of dask-kubernetes) - - git+https://github.com/dask/dask-kubernetes@b481bbbc6c1bb7d0719bc385c0537c4ba4a0ee57 + - git+https://github.com/dask/dask-kubernetes.git From 8b53d0dea499ae6296fd6bee76ca2a4383f3dac3 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Tue, 3 Dec 2019 16:31:53 -0800 Subject: [PATCH 143/156] add egg to git pip installs --- base_environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index bf24671..66f54a2 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -99,9 +99,9 @@ dependencies: - climate-toolbox - impactlab-tools - parameterize_jobs - - git+https://github.com/rhodiumgroup/rhg_compute_tools.git@allow-remote-scheduler - - git+https://github.com/NCAR/intake-esm.git + - git+https://github.com/rhodiumgroup/rhg_compute_tools.git@allow-remote-scheduler#egg=rhg_compute_tools + - git+https://github.com/NCAR/intake-esm.git#egg=intake_esm # need to install from master until 0.10.1 # due to handling of remote scheduler # (we also should at some point switch to dask-gateway instead of dask-kubernetes) - - git+https://github.com/dask/dask-kubernetes.git + - git+https://github.com/dask/dask-kubernetes.git#egg=dask_kubernetes From d0ff877903876df77e73b07390dda5f1137f9c84 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 6 Dec 2019 10:08:43 -0800 Subject: [PATCH 144/156] add geoviews dependencies --- base_environment.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base_environment.yml b/base_environment.yml index 66f54a2..8619090 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -57,6 +57,8 @@ dependencies: - numba - numcodecs - pandas + # for geoviews + - phantomjs - pip - plotly - polyline @@ -77,6 +79,8 @@ dependencies: - scikit-learn - scipy - seaborn + # for geoviews + - selenium - shapely - sparse - statsmodels From 91e8177fe0db3a60b343ff01a5deb699bcb6eb2a Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Fri, 6 Dec 2019 10:27:39 -0800 Subject: [PATCH 145/156] add iris --- base_environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/base_environment.yml b/base_environment.yml index 8619090..3825812 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -42,6 +42,7 @@ dependencies: - holoviews - h5netcdf - icu + - iris - jedi # need server proxy on workers if using remote scheduler - jupyter-server-proxy From b46e2cda7727af3eef095279da220df283b57a66 Mon Sep 17 00:00:00 2001 From: Ian Bolliger Date: Tue, 10 Dec 2019 06:14:03 -0800 Subject: [PATCH 146/156] unpin rtree after package update --- base_environment.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 3825812..9ac1e56 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -73,9 +73,7 @@ dependencies: - PyYAML - rasterio - regionmask -# the pinned rtree can be deleted from this requirements file once this is -# solved and a new release is made: https://github.com/Toblerity/rtree/issues/124 - - rtree=0.8.3 + - rtree - scikit-image - scikit-learn - scipy From d352380fb427f38b20b1b95f64a249c88a00a316 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 13 Dec 2019 15:03:09 -0800 Subject: [PATCH 147/156] pin source notebook & worker images --- notebook/Dockerfile | 4 ++-- worker/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/notebook/Dockerfile b/notebook/Dockerfile index 11c21b6..b375958 100644 --- a/notebook/Dockerfile +++ b/notebook/Dockerfile @@ -1,4 +1,4 @@ -FROM jupyter/base-notebook:latest +FROM jupyter/base-notebook:notebook-6.0.0 ARG DEBIAN_FRONTEND=noninteractive ## needed to make sure things with cython compile correctly @@ -37,7 +37,7 @@ RUN sudo chown -R $NB_USER /gcs ## more apt-get RUN sudo apt-get install -yq \ - vim \ + vim less man locate kmod dialog \ octave && \ sudo apt-get clean diff --git a/worker/Dockerfile b/worker/Dockerfile index bde5438..040a8e9 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -1,5 +1,5 @@ ## using same container as jupyter/base-notebook:python-3.7.3 -FROM ubuntu:bionic-20190612@sha256:9b1702dcfe32c873a770a32cfd306dd7fc1c4fd134adfb783db68defc8894b3c +FROM ubuntu:bionic-20190612@sha256:6e9f67fa63b0323e9a1e587fd71c561ba48a034504fb804fd26fd8800039835d ARG DEBIAN_FRONTEND=noninteractive ENV NPY_DISTUTILS_APPEND_FLAGS=1 From 74c3910ecd1c967171d8855b98ad47b8cbe6fa2a Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 13 Dec 2019 15:21:40 -0800 Subject: [PATCH 148/156] Add pip dependency to notebook_environment.yml Even though it's already installed in base_environment, we get an error because it's not in the updated environment. From https://travis-ci.org/RhodiumGroup/docker_images/jobs/624840003#L3648: Warning: you have pip-installed dependencies in your environment file, but you do not list pip itself as one of your conda dependencies. Conda may not use the correct pip to install your packages, and they may end up in the wrong place. Please add an explicit pip dependency. I'm adding one for you, but still nagging you. --- notebook/notebook_environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 3b3a322..3c312bf 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -16,6 +16,7 @@ dependencies: - octave_kernel - oct2py - openssh + - pip - pytest - pytest-cov - pytest-runner From 076f00d30a69902ae2a8d9ab417daa9e0ff92784 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 13 Dec 2019 16:25:58 -0800 Subject: [PATCH 149/156] hack to get around compiler_compat/ld bug --- common.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common.sh b/common.sh index 201ce8d..c216a88 100644 --- a/common.sh +++ b/common.sh @@ -37,3 +37,8 @@ chmod +x /usr/bin/cloud_sql_proxy chmod +x /usr/bin/prepare.sh mkdir /gcs mkdir /opt/app + +# super sketchy hack to get around our need for compiler_compat binaries and some +# other things that cause problems together? +# see https://github.com/ContinuumIO/anaconda-issues/issues/11152 +RUN -rf /opt/conda/compiler_compat/ld From f4e6b30405d3f4cdaba67083e41b6c53b0639db0 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Fri, 13 Dec 2019 16:30:42 -0800 Subject: [PATCH 150/156] oops. suuuuuper dumb bug. --- common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.sh b/common.sh index c216a88..e5f69c3 100644 --- a/common.sh +++ b/common.sh @@ -41,4 +41,4 @@ mkdir /opt/app # super sketchy hack to get around our need for compiler_compat binaries and some # other things that cause problems together? # see https://github.com/ContinuumIO/anaconda-issues/issues/11152 -RUN -rf /opt/conda/compiler_compat/ld +rm -rf /opt/conda/compiler_compat/ld From 0ea4da3673400c22a7d2f0ba02e28bcedbe6953f Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Mon, 16 Dec 2019 20:16:26 +0000 Subject: [PATCH 151/156] standardize package names --- base_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 9ac1e56..5576a49 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -70,7 +70,7 @@ dependencies: - python-blosc - python-snappy - pyviz_comms - - PyYAML + - pyyaml - rasterio - regionmask - rtree @@ -101,7 +101,7 @@ dependencies: - sidecar - climate-toolbox - impactlab-tools - - parameterize_jobs + - parameterize-jobs - git+https://github.com/rhodiumgroup/rhg_compute_tools.git@allow-remote-scheduler#egg=rhg_compute_tools - git+https://github.com/NCAR/intake-esm.git#egg=intake_esm # need to install from master until 0.10.1 From 4e8dcc287cf82f418f67ea86e09a64c982f939c9 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Mon, 16 Dec 2019 20:24:26 +0000 Subject: [PATCH 152/156] add pinning utility --- pin.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 pin.py diff --git a/pin.py b/pin.py new file mode 100644 index 0000000..c994b04 --- /dev/null +++ b/pin.py @@ -0,0 +1,153 @@ + +import click +import json +import subprocess +import re +import sys + +from ruamel.yaml import YAML + +def get_versions_in_current_environment(envname='base'): + assert re.match(r'[a-zA-Z0-9]+$', envname), ( + 'illegal environment name "{}"'.format(envname)) + + conn = subprocess.Popen( + ['conda', 'env', 'export', '-n', envname, '--json'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + o, e = conn.communicate() + if e: + raise IOError(e.decode()) + + return json.loads(o.decode()) + + +def parse_conda_dependencies(conda_dependencies): + formatted_dependencies = {'conda': {}} + for dep in conda_dependencies: + if isinstance(dep, dict): + for k, v in dep.items(): + if k not in formatted_dependencies: + formatted_dependencies[k] = {} + for subdep in v: + formatted_dependencies[k].update({subdep.split('=')[0]: subdep}) + else: + formatted_dependencies['conda'].update({dep.split('=')[0]: dep}) + + return formatted_dependencies + + +def get_pinned_version(dependency, pinned_versions): + if ('git+' in dependency) or ('http' in dependency): + return dependency + return pinned_versions.get(dependency.split('=')[0], dependency) + + +def pin_dependencies_in_conda_env_file_from_version_spec( + filepath, versions_to_pin, dry_run=False): + ''' + Parameters + ---------- + ''' + + indent_config = dict(mapping=2, sequence=2, offset=2) + + yaml = YAML(typ='rt') + yaml.indent(**indent_config) + yaml.default_flow_style = False + + with open(filepath, 'r') as f: + file_spec = yaml.load(f) + + for di, dep in enumerate(file_spec['dependencies']): + if isinstance(dep, dict): + for k, v in dep.items(): + for si, subdep in enumerate(v): + pinned = get_pinned_version(subdep, versions_to_pin[k]) + file_spec['dependencies'][di][k][si] = pinned + else: + pinned = get_pinned_version(dep, versions_to_pin['conda']) + file_spec['dependencies'][di] = pinned + + if dry_run: + sys.stdout.write("filename: {}\n{}\n".format(filepath, '-'*50)) + with YAML(output=sys.stdout) as yaml: + yaml.indent(**indent_config) + yaml.dump(file_spec) + sys.stdout.write("\n") + else: + with open(filepath, 'w+') as f: + yaml.dump(file_spec, f) + + +def pin_files(environment_files, dry_run=False): + environment_specs = {} + for envfile, envname in environment_files: + if envname not in environment_specs: + environment_specs[envname] = [] + environment_specs[envname].append(envfile) + + for envname in environment_specs: + current_versions = get_versions_in_current_environment(envname) + formatted_dependencies = parse_conda_dependencies( + current_versions.get('dependencies', [])) + + for envfile in environment_specs[envname]: + pin_dependencies_in_conda_env_file_from_version_spec( + envfile, formatted_dependencies, dry_run=dry_run) + + +@click.group() +def pinversions(): + pass + + +@pinversions.group() +def pin(): + pass + + +@pin.command() +@click.option( + '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') +def base(dry_run=False): + pin_files([('base_environment.yml', 'base')], dry_run=dry_run) + + +@pin.command() +@click.option( + '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') +def notebook(dry_run=False): + pin_files([('notebook/notebook_environment.yml', 'base')], dry_run=dry_run) + + +@pin.command() +@click.option( + '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') +def octave(dry_run=False): + pin_files([('octave-worker/octave_environment.yml', 'base')], dry_run=dry_run) + + +@pin.command() +@click.option( + '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') +def r(dry_run=False): + pin_files([('notebook/r_environment.yml', 'r')], dry_run=dry_run) + + +@pin.command() +@click.option( + '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') +def all(dry_run=False): + pin_files( + [ + ('base_environment.yml', 'base'), + ('notebook/notebook_environment.yml', 'base'), + ('octave-worker/octave_environment.yml', 'base'), + ('notebook/r_environment.yml', 'r')], + dry_run=dry_run) + + +if __name__ == "__main__": + pin() From b9689c1a151b557d91f0ae9d7347e3ddb4246e21 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Mon, 16 Dec 2019 20:38:44 +0000 Subject: [PATCH 153/156] update comments in pin.py --- pin.py | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/pin.py b/pin.py index c994b04..74fc23d 100644 --- a/pin.py +++ b/pin.py @@ -8,6 +8,19 @@ from ruamel.yaml import YAML def get_versions_in_current_environment(envname='base'): + ''' + Calls ``conda env export -n {envname} --json`` and returns spec + + Parameters + ---------- + envname : str + Name of environment to query (default 'base') + + Returns + ------- + spec : dict + Environment spec + ''' assert re.match(r'[a-zA-Z0-9]+$', envname), ( 'illegal environment name "{}"'.format(envname)) @@ -24,6 +37,9 @@ def get_versions_in_current_environment(envname='base'): def parse_conda_dependencies(conda_dependencies): + ''' + Splits conda dependencies into `conda` and other provider dependencies (e.g. `pip`) + ''' formatted_dependencies = {'conda': {}} for dep in conda_dependencies: if isinstance(dep, dict): @@ -38,7 +54,26 @@ def parse_conda_dependencies(conda_dependencies): return formatted_dependencies -def get_pinned_version(dependency, pinned_versions): +def determine_pinned_version(dependency, pinned_versions): + ''' + Handle individual packages and pinned versions to get package spec + + Defines the line-by-line logic that decides how the line from the original yaml + file and the dependencies from the local environment should be used to construct + the pinned dependency in the new spec file. + + Parameters + ---------- + dependency : str + Package name or spec from current environment file + pinned_versions : dict + Dictionary of package names and pinned versions from local environment + + Returns + ------- + pinned : str + Pinned package spec + ''' if ('git+' in dependency) or ('http' in dependency): return dependency return pinned_versions.get(dependency.split('=')[0], dependency) @@ -47,8 +82,18 @@ def get_pinned_version(dependency, pinned_versions): def pin_dependencies_in_conda_env_file_from_version_spec( filepath, versions_to_pin, dry_run=False): ''' + Pin package versions to a given spec + Parameters ---------- + filepath : str + Conda environment yml file to be pinned + versions_to_pin : dict + Dictionary of package specs, with keys package sources (e.g. ``conda``, + ``pip``), and values dictionaries of package names and pinned versions. + dry_run : bool + Print the updated environment files, rather than overwriting them. Default + False. ''' indent_config = dict(mapping=2, sequence=2, offset=2) @@ -64,10 +109,10 @@ def pin_dependencies_in_conda_env_file_from_version_spec( if isinstance(dep, dict): for k, v in dep.items(): for si, subdep in enumerate(v): - pinned = get_pinned_version(subdep, versions_to_pin[k]) + pinned = determine_pinned_version(subdep, versions_to_pin[k]) file_spec['dependencies'][di][k][si] = pinned else: - pinned = get_pinned_version(dep, versions_to_pin['conda']) + pinned = determine_pinned_version(dep, versions_to_pin['conda']) file_spec['dependencies'][di] = pinned if dry_run: @@ -82,6 +127,22 @@ def pin_dependencies_in_conda_env_file_from_version_spec( def pin_files(environment_files, dry_run=False): + ''' + Pin package versions in provided environment files + + Parameters + ---------- + environment_files : list of tuples + List of (environment file path, pin source env name) tuples to be pinned. The + second tuple element will be used as the source environment on the local + machine to look for pinned versions. + versions_to_pin : dict + Dictionary of package specs, with keys package sources (e.g. ``conda``, + ``pip``), and values dictionaries of package names and pinned versions. + dry_run : bool + Print the updated environment files, rather than overwriting them. Default + False. + ''' environment_specs = {} for envfile, envname in environment_files: if envname not in environment_specs: @@ -100,11 +161,13 @@ def pin_files(environment_files, dry_run=False): @click.group() def pinversions(): + '''View and modify package version pins''' pass @pinversions.group() def pin(): + '''Pin packages in environment files based on environments on the local machine''' pass @@ -112,6 +175,7 @@ def pin(): @click.option( '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') def base(dry_run=False): + '''Pin the base environment file''' pin_files([('base_environment.yml', 'base')], dry_run=dry_run) @@ -119,6 +183,7 @@ def base(dry_run=False): @click.option( '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') def notebook(dry_run=False): + '''Pin the notebook base environment file''' pin_files([('notebook/notebook_environment.yml', 'base')], dry_run=dry_run) @@ -126,6 +191,7 @@ def notebook(dry_run=False): @click.option( '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') def octave(dry_run=False): + '''Pin the worker octave environment files''' pin_files([('octave-worker/octave_environment.yml', 'base')], dry_run=dry_run) @@ -133,6 +199,7 @@ def octave(dry_run=False): @click.option( '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') def r(dry_run=False): + '''Pin the notebook r environment files''' pin_files([('notebook/r_environment.yml', 'r')], dry_run=dry_run) @@ -140,6 +207,7 @@ def r(dry_run=False): @click.option( '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') def all(dry_run=False): + '''Pin all environment files''' pin_files( [ ('base_environment.yml', 'base'), From 104e428ec8e681c197091e91ef54f6c1f74e72a6 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Mon, 16 Dec 2019 20:39:02 +0000 Subject: [PATCH 154/156] pin all versions --- base_environment.yml | 174 +++++++++++++-------------- notebook/notebook_environment.yml | 42 +++---- notebook/r_environment.yml | 38 +++--- octave-worker/octave_environment.yml | 4 +- 4 files changed, 129 insertions(+), 129 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 5576a49..701614a 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -2,106 +2,106 @@ name: base channels: - conda-forge dependencies: - - beautifulsoup4 - - boto3 - - bqplot - - bumpversion - - cachecontrol - - cartopy - - cftime - - click - - compilers - - dask - - dask-glm - - dask-ml - - datashader - - distributed - - dropbox + - beautifulsoup4=4.8.1=py37_0 + - boto3=1.10.34=py_0 + - bqplot=0.12.1=py_0 + - bumpversion=0.5.3=py_1001 + - cachecontrol=0.12.5=py_0 + - cartopy=0.17.0=py37h423102d_1009 + - cftime=1.0.4.2=py37hc1659b7_0 + - click=7.0=py_0 + - compilers=1.0.4=0 + - dask=2.8.1=py_0 + - dask-glm=0.2.0=py_1 + - dask-ml=1.1.1=py_0 + - datashader=0.8.0=py_0 + - distributed=2.8.1=py_0 + - dropbox=9.4.0=py_0 # need to make sure we get esmpy compiled with mpi otherwise xesmf regridding # won't work - esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101 - - fastparquet - - fiona - - fusepy + - fastparquet=0.3.2=py37hc1659b7_0 + - fiona=1.8.13=py37h900e953_0 + - fusepy=3.0.1=py_0 # this gcc pin is necessary b/c of a weird feature in the h553295d_15 build # which makes it hard to build numpy-based cython extensions (like pyclaw). # we should try removing it whenever we next do an update and see if Clawpack # can still be built - gcc_linux-64=7.3.0=h553295d_14 - - gcsfs - - gdal - - geoalchemy2 - - geopandas - - geopy - - geotiff - - geoviews - - git - - gitpython - - google-cloud-container - - google-cloud-storage - - holoviews - - h5netcdf - - icu - - iris - - jedi + - gcsfs=0.5.3=py_0 + - gdal=3.0.2=py37hbb6b9fb_5 + - geoalchemy2=0.6.3=py_0 + - geopandas=0.6.2=py_0 + - geopy=1.20.0=py_0 + - geotiff=1.5.1=hbd99317_7 + - geoviews=1.6.6=py_0 + - git=2.24.0=pl526hce37bd2_1 + - gitpython=3.0.5=py_0 + - google-cloud-container=0.3.0=py37_0 + - google-cloud-storage=1.23.0=py37_0 + - holoviews=1.12.7=py_0 + - h5netcdf=0.7.4=py_0 + - icu=64.2=he1b5a44_1 + - iris=2.2.0=py37_1003 + - jedi=0.15.1=py37_0 # need server proxy on workers if using remote scheduler - - jupyter-server-proxy + - jupyter-server-proxy=1.2.0=py_0 - kubernetes - - lapack - - lz4 - - make - - matplotlib - - nc-time-axis - - ncurses - - netcdf-fortran - - netcdf4 - - numba - - numcodecs - - pandas + - lapack=3.6.1=ha44fe06_2 + - lz4=2.2.1=py37hd79334b_0 + - make=4.2.1=h14c3975_2004 + - matplotlib=3.1.2=py37_1 + - nc-time-axis=1.2.0=py_0 + - ncurses=6.1=hf484d3e_1002 + - netcdf-fortran=4.5.2=mpi_mpich_ha8580a0_2 + - netcdf4=1.5.3=mpi_mpich_py37h01ee55b_1 + - numba=0.46.0=py37hb3f55d8_1 + - numcodecs=0.6.4=py37he1b5a44_0 + - pandas=0.25.3=py37hb3f55d8_0 # for geoviews - - phantomjs - - pip - - plotly - - polyline - - pygeos - - pyinterp - - pyshp - - python - - python-blosc - - python-snappy - - pyviz_comms - - pyyaml - - rasterio - - regionmask - - rtree - - scikit-image - - scikit-learn - - scipy - - seaborn + - phantomjs=2.1.1=1 + - pip=19.3.1=py37_0 + - plotly=4.3.0=py_0 + - polyline=1.4.0=py_0 + - pygeos=0.5=py37h5d51c17_1 + - pyinterp=0.0.7=py37h97f2665_0 + - pyshp=2.1.0=py_0 + - python=3.7.3=h357f687_2 + - python-blosc=1.8.1=py37hf484d3e_0 + - python-snappy=0.5.4=py37hee44bf9_1 + - pyviz_comms=0.7.2=py_0 + - pyyaml=5.2=py37h516909a_0 + - rasterio=1.1.1=py37h900e953_0 + - regionmask=0.4.0=py_0 + - rtree=0.8.3=py37h7b0cdae_1003 + - scikit-image=0.16.2=py37hb3f55d8_0 + - scikit-learn=0.22=py37hcdab131_0 + - scipy=1.3.2=py37h921218d_0 + - seaborn=0.9.0=py_2 # for geoviews - - selenium - - shapely - - sparse - - statsmodels - - tini - - unzip - - uritemplate - - xarray - - xesmf - - xgcm - - xhistogram - - xlrd - - xrft - - zarr - - zeromq - - zict + - selenium=3.141.0=py37h516909a_1000 + - shapely=1.6.4=py37h5d51c17_1007 + - sparse=0.8.0=py_0 + - statsmodels=0.10.2=py37hc1659b7_0 + - tini=0.18.0=h14c3975_1001 + - unzip=6.0=h516909a_0 + - uritemplate=3.0.0=py_1 + - xarray=0.14.1=py_0 + - xesmf=0.2.1=py_0 + - xgcm=0.2.0=py_0 + - xhistogram=0.1.1=py_0 + - xlrd=1.2.0=py_0 + - xrft=0.2.0=py_0 + - zarr=2.3.2=py37_0 + - zeromq=4.3.2=he1b5a44_2 + - zict=1.0.0=py_0 - pip: - - mapbox - - py-noaa - - sidecar - - climate-toolbox - - impactlab-tools - - parameterize-jobs + - mapbox==0.18.0 + - py-noaa==1.0 + - sidecar==0.3.0 + - climate-toolbox==0.1.5 + - impactlab-tools==0.4.0 + - parameterize-jobs==0.1.1 - git+https://github.com/rhodiumgroup/rhg_compute_tools.git@allow-remote-scheduler#egg=rhg_compute_tools - git+https://github.com/NCAR/intake-esm.git#egg=intake_esm # need to install from master until 0.10.1 diff --git a/notebook/notebook_environment.yml b/notebook/notebook_environment.yml index 3c312bf..81d85f6 100644 --- a/notebook/notebook_environment.yml +++ b/notebook/notebook_environment.yml @@ -2,26 +2,26 @@ name: base channels: - conda-forge dependencies: - - black - - coverage - - dask-labextension - - flake8 - - ipdb - - ipyleaflet - - ipympl - - jupyterlab_code_formatter - - nano - - nb_conda_kernels - - nose - - octave_kernel - - oct2py - - openssh - - pip - - pytest - - pytest-cov - - pytest-runner - - python-graphviz - - sphinx - - tox + - black=19.10b0=py37_0 + - coverage=4.5.4=py37h516909a_0 + - dask-labextension=1.0.3=py_0 + - flake8=3.7.9=py37_0 + - ipdb=0.12.3=py_0 + - ipyleaflet=0.11.6=py37_0 + - ipympl=0.3.3=py_0 + - jupyterlab_code_formatter=0.7.0=py_0 + - nano=2.9.8=hb256ff8_1000 + - nb_conda_kernels=2.2.2=py37_0 + - nose=1.3.7=py37_1003 + - octave_kernel=0.31.0=py_0 + - oct2py=5.0.4=py_0 + - openssh=7.9p1=h0fa992c_1 + - pip=19.3.1=py37_0 + - pytest=5.3.1=py37_0 + - pytest-cov=2.8.1=py_0 + - pytest-runner=5.2=py_0 + - python-graphviz=0.13.2=py_0 + - sphinx=2.2.2=py_0 + - tox=3.14.2=py_0 - pip: - black_nbconvert diff --git a/notebook/r_environment.yml b/notebook/r_environment.yml index 6cf3c33..09e3e50 100644 --- a/notebook/r_environment.yml +++ b/notebook/r_environment.yml @@ -3,22 +3,22 @@ channels: - conda-forge - r dependencies: - - rpy2 - - r-base - - r-irkernel - - r-plyr - - r-devtools - - r-tidyverse - - r-shiny - - r-rmarkdown - - r-forecast - - r-rsqlite - - r-reshape2 - - r-nycflights13 - - r-caret - - r-rcurl - - r-crayon - - r-randomforest - - r-sparklyr - - r-htmlwidgets - - r-hexbin + - rpy2=3.1.0=py38r36hc1659b7_3 + - r-base=3.6.1=h3a67422_6 + - r-irkernel=1.1=r36h6115d3f_0 + - r-plyr=1.8.4=r36h0357c0b_1003 + - r-devtools=2.2.1=r36h6115d3f_0 + - r-tidyverse=1.3.0=r36h6115d3f_0 + - r-shiny=1.4.0=r36h6115d3f_0 + - r-rmarkdown=1.18=r36h6115d3f_0 + - r-forecast=8.10=r36h0357c0b_0 + - r-rsqlite=2.1.4=r36h0357c0b_0 + - r-reshape2=1.4.3=r36h0357c0b_1004 + - r-nycflights13=1.0.1=r36h6115d3f_0 + - r-caret=6.0_84=r36hcdcec82_1 + - r-rcurl=1.95_4.12=r36hcdcec82_1 + - r-crayon=1.3.4=r36h6115d3f_1002 + - r-randomforest=4.6_14=r36h9bbef5b_1002 + - r-sparklyr=1.0.5=r36h6115d3f_0 + - r-htmlwidgets=1.5.1=r36h6115d3f_0 + - r-hexbin=1.28.0=r36h9bbef5b_0 diff --git a/octave-worker/octave_environment.yml b/octave-worker/octave_environment.yml index 03fc43e..704df48 100644 --- a/octave-worker/octave_environment.yml +++ b/octave-worker/octave_environment.yml @@ -2,5 +2,5 @@ name: base channels: - conda-forge dependencies: - - octave_kernel - - oct2py + - octave_kernel=0.31.0=py_0 + - oct2py=5.0.4=py_0 From 7950c13d2551b6c8603a446d5d49326c670118b4 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Mon, 16 Dec 2019 21:07:37 +0000 Subject: [PATCH 155/156] update to allow comments for round-trip unpinning --- base_environment.yml | 6 +++--- pin.py | 25 +++++++++++++++++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index 701614a..f134002 100644 --- a/base_environment.yml +++ b/base_environment.yml @@ -19,7 +19,7 @@ dependencies: - dropbox=9.4.0=py_0 # need to make sure we get esmpy compiled with mpi otherwise xesmf regridding # won't work - - esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101 + - esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101 # pinkeep: esmpy=8.0.0=mpi_mpich_py37ha9b28fa_101 - fastparquet=0.3.2=py37hc1659b7_0 - fiona=1.8.13=py37h900e953_0 - fusepy=3.0.1=py_0 @@ -27,7 +27,7 @@ dependencies: # which makes it hard to build numpy-based cython extensions (like pyclaw). # we should try removing it whenever we next do an update and see if Clawpack # can still be built - - gcc_linux-64=7.3.0=h553295d_14 + - gcc_linux-64=7.3.0=h553295d_14 # pinkeep: gcc_linux-64=7.3.0=h553295d_14 - gcsfs=0.5.3=py_0 - gdal=3.0.2=py37hbb6b9fb_5 - geoalchemy2=0.6.3=py_0 @@ -66,7 +66,7 @@ dependencies: - pygeos=0.5=py37h5d51c17_1 - pyinterp=0.0.7=py37h97f2665_0 - pyshp=2.1.0=py_0 - - python=3.7.3=h357f687_2 + - python=3.7.3=h357f687_2 # pinkeep: python=3.7 - python-blosc=1.8.1=py37hf484d3e_0 - python-snappy=0.5.4=py37hee44bf9_1 - pyviz_comms=0.7.2=py_0 diff --git a/pin.py b/pin.py index 74fc23d..a9efdcb 100644 --- a/pin.py +++ b/pin.py @@ -73,10 +73,18 @@ def determine_pinned_version(dependency, pinned_versions): ------- pinned : str Pinned package spec + comment : str or None + Comment to include in file if a pinning flag should be included ''' if ('git+' in dependency) or ('http' in dependency): - return dependency - return pinned_versions.get(dependency.split('=')[0], dependency) + return dependency, None + + if '=' in dependency: + comment = 'pinkeep: {}'.format(dependency) + else: + comment = None + + return pinned_versions.get(dependency.split('=')[0], dependency), comment def pin_dependencies_in_conda_env_file_from_version_spec( @@ -109,11 +117,20 @@ def pin_dependencies_in_conda_env_file_from_version_spec( if isinstance(dep, dict): for k, v in dep.items(): for si, subdep in enumerate(v): - pinned = determine_pinned_version(subdep, versions_to_pin[k]) + pinned, comment = determine_pinned_version( + subdep, versions_to_pin[k]) + file_spec['dependencies'][di][k][si] = pinned + if comment is not None: + file_spec['dependencies'][di][k].yaml_add_eol_comment( + comment, si) else: - pinned = determine_pinned_version(dep, versions_to_pin['conda']) + pinned, comment = determine_pinned_version(dep, versions_to_pin['conda']) file_spec['dependencies'][di] = pinned + + if comment is not None: + file_spec['dependencies'].yaml_add_eol_comment( + comment, di) if dry_run: sys.stdout.write("filename: {}\n{}\n".format(filepath, '-'*50)) From ff5a13a21e1950ed555912a3a9e748e0fdc20459 Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Mon, 16 Dec 2019 22:25:24 +0000 Subject: [PATCH 156/156] add unpin command and improve documentation --- pin.py | 212 ++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 165 insertions(+), 47 deletions(-) diff --git a/pin.py b/pin.py index a9efdcb..175ab22 100644 --- a/pin.py +++ b/pin.py @@ -1,3 +1,20 @@ +''' +Functions and command line app for manipulating conda environment specs + +Usage +----- + +.. code-block:: bash + + $ python pin.py pin all + + $ python pin.py unpin all + +Use the ``--dry-run`` flag to see the effect of any commands rather than modifying the +environment files directly. The ``--help`` command provides additional information about +each command. + +''' import click import json @@ -7,6 +24,7 @@ from ruamel.yaml import YAML + def get_versions_in_current_environment(envname='base'): ''' Calls ``conda env export -n {envname} --json`` and returns spec @@ -54,7 +72,7 @@ def parse_conda_dependencies(conda_dependencies): return formatted_dependencies -def determine_pinned_version(dependency, pinned_versions): +def _determine_pinned_version(dependency, pinned_versions): ''' Handle individual packages and pinned versions to get package spec @@ -117,7 +135,7 @@ def pin_dependencies_in_conda_env_file_from_version_spec( if isinstance(dep, dict): for k, v in dep.items(): for si, subdep in enumerate(v): - pinned, comment = determine_pinned_version( + pinned, comment = _determine_pinned_version( subdep, versions_to_pin[k]) file_spec['dependencies'][di][k][si] = pinned @@ -125,7 +143,7 @@ def pin_dependencies_in_conda_env_file_from_version_spec( file_spec['dependencies'][di][k].yaml_add_eol_comment( comment, si) else: - pinned, comment = determine_pinned_version(dep, versions_to_pin['conda']) + pinned, comment = _determine_pinned_version(dep, versions_to_pin['conda']) file_spec['dependencies'][di] = pinned if comment is not None: @@ -153,9 +171,6 @@ def pin_files(environment_files, dry_run=False): List of (environment file path, pin source env name) tuples to be pinned. The second tuple element will be used as the source environment on the local machine to look for pinned versions. - versions_to_pin : dict - Dictionary of package specs, with keys package sources (e.g. ``conda``, - ``pip``), and values dictionaries of package names and pinned versions. dry_run : bool Print the updated environment files, rather than overwriting them. Default False. @@ -176,63 +191,166 @@ def pin_files(environment_files, dry_run=False): envfile, formatted_dependencies, dry_run=dry_run) -@click.group() -def pinversions(): - '''View and modify package version pins''' - pass +def _unpin_dependency(tree, key): + ''' + Determines the unpinned spec for individual packages based on comments and spec + ''' + ct = tree.ca.items.get(key) + if ct: + comment = ct[0].value.strip() + if 'pinkeep:' in comment: + pinned = comment.split('pinkeep:')[1].strip() + del tree.ca.items[key] + return pinned -@pinversions.group() -def pin(): - '''Pin packages in environment files based on environments on the local machine''' - pass + pkg = tree[key] + if ('http:' in pkg) or ('https:' in pkg) or ('git+' in pkg) or ('ssh+' in pkg): + return pkg -@pin.command() -@click.option( - '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') -def base(dry_run=False): - '''Pin the base environment file''' - pin_files([('base_environment.yml', 'base')], dry_run=dry_run) + return pkg.split('=')[0] -@pin.command() -@click.option( - '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') -def notebook(dry_run=False): - '''Pin the notebook base environment file''' - pin_files([('notebook/notebook_environment.yml', 'base')], dry_run=dry_run) +def unpin_dependencies_in_conda_env_file(filepath, dry_run=False): + ''' + Un-pin dependencies in conda environment file + + If encounters dependencies with ``# pinkeep: pkg=vers`` directives, these are + preserved verbatim in the final spec. + + Paramters + --------- + filepath : str + Path to the environment file to unpin + dry_run : bool, optional + Print rather than modify the environment file + ''' + + indent_config = dict(mapping=2, sequence=2, offset=2) + + yaml = YAML(typ='rt') + yaml.indent(**indent_config) + yaml.default_flow_style = False + with open(filepath, 'r') as f: + file_spec = yaml.load(f) -@pin.command() -@click.option( - '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') -def octave(dry_run=False): - '''Pin the worker octave environment files''' - pin_files([('octave-worker/octave_environment.yml', 'base')], dry_run=dry_run) + for di, dep in enumerate(file_spec['dependencies']): + if isinstance(dep, dict): + for k, v in dep.items(): + for si, subdep in enumerate(v): + file_spec['dependencies'][di][k][si] = _unpin_dependency( + file_spec['dependencies'][di][k], si) + else: + file_spec['dependencies'][di] = _unpin_dependency( + file_spec['dependencies'], di) + + if dry_run: + sys.stdout.write("filename: {}\n{}\n".format(filepath, '-'*50)) + with YAML(output=sys.stdout) as yaml: + yaml.indent(**indent_config) + yaml.dump(file_spec) + sys.stdout.write("\n") + else: + with open(filepath, 'w+') as f: + yaml.dump(file_spec, f) -@pin.command() -@click.option( - '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') -def r(dry_run=False): - '''Pin the notebook r environment files''' - pin_files([('notebook/r_environment.yml', 'r')], dry_run=dry_run) +def unpin_files(environment_files, dry_run=False): + ''' + Unpin package versions in provided environment files + + Parameters + ---------- + environment_files : list of tuples + List of (environment file path, pin source env name) tuples to be pinned. The + second tuple element will be used as the source environment on the local + machine to look for pinned versions. + dry_run : bool + Print the updated environment files, rather than overwriting them. Default + False. + ''' + for envfile, envname in environment_files: + unpin_dependencies_in_conda_env_file(envfile, dry_run=dry_run) -@pin.command() + +@click.group() +def pinversions(): + '''View and modify package version pins''' + pass + + +@pinversions.command() +@click.argument( + 'file', default='all') +@click.option( + '--dry-run', + is_flag=True, + default=False, + help='print proposed spec rather than modifying it', +) +def pin(file, dry_run): + '''Pin packages in environment files based on environments on the local machine''' + + spec_files = [ + ('base_environment.yml', 'base'), + ('notebook/notebook_environment.yml', 'base'), + ('octave-worker/octave_environment.yml', 'base'), + ('notebook/r_environment.yml', 'r')] + + if file == 'all': + pin_files(spec_files, dry_run=dry_run) + elif file == 'base': + pin_files([spec_files[0]], dry_run=dry_run) + elif file == 'notebook': + pin_files([spec_files[1]], dry_run=dry_run) + elif file == 'octave': + pin_files([spec_files[2]], dry_run=dry_run) + elif file == 'r': + pin_files([spec_files[3]], dry_run=dry_run) + else: + raise ValueError( + 'env type not recognized: {}' + 'choose from "base", "notebook", "octave", "r", or "all".' + .format(file)) + + +@pinversions.command() +@click.argument( + 'file', default='all') @click.option( - '--dry-run', is_flag=True, default=False, help='print proposed spec rather than modifying it') -def all(dry_run=False): - '''Pin all environment files''' - pin_files( - [ + '--dry-run', + is_flag=True, + default=False, + help='print proposed spec rather than modifying it', +) +def unpin(file, dry_run): + '''Unpin packages in environment files''' + + spec_files = [ ('base_environment.yml', 'base'), ('notebook/notebook_environment.yml', 'base'), ('octave-worker/octave_environment.yml', 'base'), - ('notebook/r_environment.yml', 'r')], - dry_run=dry_run) + ('notebook/r_environment.yml', 'r')] + + if file == 'all': + unpin_files(spec_files, dry_run=dry_run) + elif file == 'base': + unpin_files([spec_files[0]], dry_run=dry_run) + elif file == 'notebook': + unpin_files([spec_files[1]], dry_run=dry_run) + elif file == 'octave': + unpin_files([spec_files[2]], dry_run=dry_run) + elif file == 'r': + unpin_files([spec_files[3]], dry_run=dry_run) + else: + raise ValueError( + 'env type not recognized: {}' + 'choose from "base", "notebook", "octave", "r", or "all".' + .format(file)) if __name__ == "__main__": - pin() + pinversions()