From c10906d36243bfc9444cec53930f3ea0474817f1 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 12 Sep 2024 13:14:53 -0700 Subject: [PATCH] moving julia to a new image repo --- .circleci/config.yml | 25 ----- deployments/julia/hubploy.yaml | 7 +- deployments/julia/image/Dockerfile | 94 ------------------- deployments/julia/image/README.md | 5 + deployments/julia/image/apt.txt | 16 ---- deployments/julia/image/environment.yml | 28 ------ .../julia/image/infra-requirements.txt | 29 ------ .../julia/image/install-julia-packages.jl | 30 ------ deployments/julia/image/install-julia.bash | 11 --- .../julia/image/install-mambaforge.bash | 42 --------- scripts/user-image-management/repos.txt | 1 + 11 files changed, 7 insertions(+), 281 deletions(-) delete mode 100644 deployments/julia/image/Dockerfile create mode 100644 deployments/julia/image/README.md delete mode 100644 deployments/julia/image/apt.txt delete mode 100644 deployments/julia/image/environment.yml delete mode 100644 deployments/julia/image/infra-requirements.txt delete mode 100755 deployments/julia/image/install-julia-packages.jl delete mode 100755 deployments/julia/image/install-julia.bash delete mode 100755 deployments/julia/image/install-mambaforge.bash diff --git a/.circleci/config.yml b/.circleci/config.yml index baf7d6cec..0ac875604 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -229,12 +229,6 @@ jobs: hubploy deploy --timeout 30m ischool hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy julia - command: | - hubploy deploy --timeout 30m julia hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy prob140 command: | @@ -311,15 +305,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: julia - name: julia hub image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod deploy: jobs: @@ -376,15 +361,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: julia - name: julia hub image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging # Build images only during the staging deploy. All merges # to prod need to go via staging, so prod should *never* # use images not built for staging. By enforcing this at the @@ -397,7 +373,6 @@ workflows: - datahub image build - edx image build - ischool image build - - julia hub image build filters: branches: diff --git a/deployments/julia/hubploy.yaml b/deployments/julia/hubploy.yaml index d93ba8880..36d995289 100644 --- a/deployments/julia/hubploy.yaml +++ b/deployments/julia/hubploy.yaml @@ -1,10 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/julia-user-image - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/julia-user-image:PLACEHOLDER cluster: provider: gcloud diff --git a/deployments/julia/image/Dockerfile b/deployments/julia/image/Dockerfile deleted file mode 100644 index d4095a73a..000000000 --- a/deployments/julia/image/Dockerfile +++ /dev/null @@ -1,94 +0,0 @@ -FROM buildpack-deps:jammy-scm - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV DEBIAN_FRONTEND=noninteractive -ENV NB_USER jovyan -ENV NB_UID 1000 -ENV SHELL /bin/bash - -ENV CONDA_DIR /srv/conda -ENV JULIA_DIR /srv/julia - -ENV PATH ${JULIA_DIR}/bin:${CONDA_DIR}/bin:$PATH - -ENV TZ=America/Los_Angeles -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -RUN adduser --disabled-password --gecos "Default Jupyter user" ${NB_USER} - -# Useful utils that folks sort of take for granted -RUN apt-get update -qq --yes && \ - apt-get install --yes -qq \ - less \ - htop \ - man \ - nano \ - screen \ - tar \ - tmux \ - wget \ - vim \ - tini \ - build-essential \ - locales \ - dbus-x11 \ - xorg \ - xubuntu-icon-theme \ - xfce4 \ - > /dev/null - -RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ - locale-gen - -# Install all apt packages -COPY apt.txt /tmp/apt.txt -RUN apt-get -qq update --yes && \ - apt-get -qq install --yes --no-install-recommends \ - $(grep -v ^# /tmp/apt.txt) && \ - apt-get -qq purge && \ - apt-get -qq clean && \ - rm -rf /var/lib/apt/lists/* - -RUN mkdir -p ${JULIA_DIR} && chown ${NB_USER}:${NB_USER} ${JULIA_DIR} - -WORKDIR /home/jovyan - -COPY install-mambaforge.bash /tmp/install-mambaforge.bash -RUN /tmp/install-mambaforge.bash - -USER ${NB_USER} - -COPY environment.yml /tmp/environment.yml -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && mamba clean -afy - -COPY infra-requirements.txt /tmp/infra-requirements.txt -RUN pip install --no-cache -r /tmp/infra-requirements.txt - -USER root -ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} -RUN playwright install-deps -RUN chown -Rh jovyan:jovyan /srv/conda - -USER ${NB_USER} - -# DH-330, similar to DH-164 -ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} -RUN playwright install chromium - -# 2024-01-13 sknapp: incompatible due to notebook 7 -# RUN jupyter contrib nbextensions install --sys-prefix --symlink && \ -# jupyter nbextensions_configurator enable --sys-prefix - -COPY install-julia.bash /tmp/install-julia.bash -RUN /tmp/install-julia.bash - -ENV JULIA_DEPOT_PATH ${JULIA_DIR}/pkg - -RUN JUPYTER_DATA_DIR=${CONDA_DIR}/share/jupyter julia -e 'using Pkg; Pkg.add("IJulia"); using IJulia; installkernel("Julia");' - -COPY install-julia-packages.jl /tmp/install-julia-packages.jl -RUN /tmp/install-julia-packages.jl - -ENTRYPOINT ["tini", "--"] diff --git a/deployments/julia/image/README.md b/deployments/julia/image/README.md new file mode 100644 index 000000000..dd9bbe5ce --- /dev/null +++ b/deployments/julia/image/README.md @@ -0,0 +1,5 @@ +# Julia Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/julia-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/julia-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/julia/image/apt.txt b/deployments/julia/image/apt.txt deleted file mode 100644 index 2a0dd4de6..000000000 --- a/deployments/julia/image/apt.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Dependencies for nbconvert -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# https://github.com/berkeley-dsep-infra/datahub/issues/3719 -texlive-lang-chinese -lmodern - -# Other useful document-related tools -pandoc -latexdiff - -# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 -strace -libnss3 -libnspr4 diff --git a/deployments/julia/image/environment.yml b/deployments/julia/image/environment.yml deleted file mode 100644 index 13ea0630e..000000000 --- a/deployments/julia/image/environment.yml +++ /dev/null @@ -1,28 +0,0 @@ -dependencies: -- jupyter-server-proxy==4.2.0 -- nodejs==20.8.1 -- pip==22.3.1 -- python==3.11.* -- requests==2.31.0 - -- syncthing==1.18.6 - -# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 -- traitlets=5.9.* - -# from requirements.txt -- matplotlib==3.7.2 - -# Required for issue #3057 -- xlrd==2.0.1 - -# For https://github.com/berkeley-dsep-infra/datahub/issues/3079 -- websockify -- pip: - - notebook==7.0.7 - - jupyter-desktop-server - - otter-grader==3.1.4 - # for notebook exporting - - nbconvert[webpdf]==7.16.4 - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 diff --git a/deployments/julia/image/infra-requirements.txt b/deployments/julia/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/julia/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/julia/image/install-julia-packages.jl b/deployments/julia/image/install-julia-packages.jl deleted file mode 100755 index c30f854eb..000000000 --- a/deployments/julia/image/install-julia-packages.jl +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env julia -using Pkg; - -Pkg.add.([ - Pkg.PackageSpec(;name="FFTW", version="1.7.1"), - Pkg.PackageSpec(;name="FastGaussQuadrature", version="1.0.0"), - Pkg.PackageSpec(;name="ForwardDiff", version="0.10.36"), - Pkg.PackageSpec(;name="NBInclude", version="2.3.0"), - Pkg.PackageSpec(;name="Delaunator", version="0.1.1"), - Pkg.PackageSpec(;name="TriplotRecipes", version="0.1.2"), - Pkg.PackageSpec(;name="StaticArrays", version="1.6.2"), - Pkg.PackageSpec(;name="ForwardDiff", version="0.10.35"), - Pkg.PackageSpec(;name="BenchmarkTools", version="1.3.2"), - Pkg.PackageSpec(;name="Query", version="1.0.0"), - Pkg.PackageSpec(;name="VegaLite", version="2.6.0"), - Pkg.PackageSpec(;name="CSVFiles", version="1.0.1"), - Pkg.PackageSpec(;name="Distributions", version="0.23.11"), - Pkg.PackageSpec(;name="DataFrames"), - Pkg.PackageSpec(;name="Plots", version="1.24.3"), - Pkg.PackageSpec(;name="Images", version="0.24.1"), - Pkg.PackageSpec(;name="PyPlot", version="2.10.0"), - Pkg.PackageSpec(;name="PyCall", version="1.92.5"), - Pkg.PackageSpec(;name="Roots", version="1.3.11"), - Pkg.PackageSpec(;name="Polynomials", version="2.0.18"), - Pkg.PackageSpec(;name="DifferentialEquations", version="7.8.0"), - Pkg.PackageSpec(;name="Optim", version="1.5.0") -]); - -# Precompiling installed packages -Pkg.precompile(); diff --git a/deployments/julia/image/install-julia.bash b/deployments/julia/image/install-julia.bash deleted file mode 100755 index f8b340c82..000000000 --- a/deployments/julia/image/install-julia.bash +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -JULIA_MAJOR_VERSION="1" -JULIA_MINOR_VERSION="9" -JULIA_PATCH_VERSION="2" - -JULIA_MAIN_VERSION="${JULIA_MAJOR_VERSION}.${JULIA_MINOR_VERSION}" -JULIA_FULL_VERSION="${JULIA_MAIN_VERSION}.${JULIA_PATCH_VERSION}" - -curl --silent --location --fail https://julialang-s3.julialang.org/bin/linux/x64/${JULIA_MAIN_VERSION}/julia-${JULIA_FULL_VERSION}-linux-x86_64.tar.gz | tar xvz -C ${JULIA_DIR} --strip-components=1 diff --git a/deployments/julia/image/install-mambaforge.bash b/deployments/julia/image/install-mambaforge.bash deleted file mode 100755 index 200691aa7..000000000 --- a/deployments/julia/image/install-mambaforge.bash +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# This downloads and installs a pinned version of mambaforge -set -ex - -cd $(dirname $0) -MAMBAFORGE_VERSION='24.3.0-0' - -URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/mambaforge-installer.sh - -# make sure we don't do anything funky with user's $HOME -# since this is run as root -unset HOME - -wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} - -bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" - -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# empty conda history file, -# which seems to result in some effective pinning of packages in the initial env, -# which we don't intend. -# this file must not be *removed*, however -echo '' > ${CONDA_DIR}/conda-meta/history - -# Clean things out! -conda clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} - -# Remove the pip cache created as part of installing mambaforge -rm -rf /root/.cache - -chown -R $NB_USER:$NB_USER ${CONDA_DIR} - -conda list -n root diff --git a/scripts/user-image-management/repos.txt b/scripts/user-image-management/repos.txt index 78f88aff0..e7e40caa8 100644 --- a/scripts/user-image-management/repos.txt +++ b/scripts/user-image-management/repos.txt @@ -6,6 +6,7 @@ git@github.com:berkeley-dsep-infra/data101-user-image.git git@github.com:berkeley-dsep-infra/dev-primary-image.git git@github.com:berkeley-dsep-infra/dev-secondary-image.git git@github.com:berkeley-dsep-infra/eecs-user-image.git +git@github.com:berkeley-dsep-infra/julia-user-image.git git@github.com:berkeley-dsep-infra/logodev-user-image.git git@github.com:berkeley-dsep-infra/nature-user-image.git git@github.com:berkeley-dsep-infra/publichealth-user-image.git