From bb8532dce164b83bd76d49a4449aa7a3cd188f1c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 13 Aug 2024 10:52:36 -0700 Subject: [PATCH 1/5] preparing for more hubploy testing --- deployments/logodev/hubploy.yaml | 5 +- deployments/logodev/image/apt.txt | 93 ------------------- deployments/logodev/image/environment.yml | 65 ------------- .../logodev/image/infra-requirements.txt | 29 ------ deployments/logodev/image/postBuild | 6 -- 5 files changed, 1 insertion(+), 197 deletions(-) delete mode 100644 deployments/logodev/image/apt.txt delete mode 100644 deployments/logodev/image/environment.yml delete mode 100644 deployments/logodev/image/infra-requirements.txt delete mode 100644 deployments/logodev/image/postBuild diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index 7760f94a1..36e817577 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,10 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a02e39fbaedb - path: image/ - repo2docker: - base_image: docker.io/library/buildpack-deps:jammy + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:829f9eaececd registry: provider: gcloud gcloud: diff --git a/deployments/logodev/image/apt.txt b/deployments/logodev/image/apt.txt deleted file mode 100644 index a6c4589ed..000000000 --- a/deployments/logodev/image/apt.txt +++ /dev/null @@ -1,93 +0,0 @@ -# Some linux packages for basic terminal work, particularly -# oriented at users new to Unix/cmd line environments. - -# installing less as more just isn't enough -less - -# Basic unix tools -man -man-db -manpages-posix -manpages-dev -manpages-posix-dev - -# Download tools -curl -wget - -# Core text editors on a *nix box: vim -vim - -# A couple of CLI editors that are easier than vim -# micro # currently not working on 18.04 -nano -jed -jed-extra - -# powerful terminal-based file manager, better than the one in JLab -mc - -# for easily managing multiple repositories with one command (perl-doc -# is needed for its help pages to work) -mr -perl-doc - -# Regular build tools for compiling common stuff -build-essential -gfortran - -# Dependencies for nbconvert -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# https://github.com/berkeley-dsep-infra/datahub/issues/3719 -texlive-lang-chinese -lmodern - -# Other useful document-related tools -pandoc -latexdiff - -# Some useful git utilities use basic Ruby -ruby - -# Other niceties for command-line work and life -ack # powerful grep-like tool -pydf # colorized disk usage -tmux -screen -htop -nnn # cmd line file manager -zsh -rsync -tig # console UI for git -multitail - -# For later, these are not available in 18.04 -#browsh # text-based web browser, occasionally handy -#dasel # json/yml/csv/etc data wrangling at the terminal -#fzf # fuzzy file finder - -## This section adds tools for desktop environment usage -dbus-x11 -xorg -xubuntu-icon-theme -xfce4 -xfce4-goodies -xclip -xsel -firefox -chromium-browser - -# GUI text editors -vim-gtk3 -gedit - -# Git clients and tools -git-gui -gitg -qgit -meld - -# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 -zip diff --git a/deployments/logodev/image/environment.yml b/deployments/logodev/image/environment.yml deleted file mode 100644 index 386806bea..000000000 --- a/deployments/logodev/image/environment.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: data100-FA23 - -channels: -- conda-forge -- pytorch - -dependencies: -- python==3.11.* -- git==2.39.1 -- jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.11 -- jupyterlab-favorites==3.0.0 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 -- nbgitpuller==1.2.1 -- notebook==7.0.7 -- folium==0.14.0 -- h5netcdf==1.0.2 -- ipywidgets==8.0.7 -- jupysql==0.8.0 -- jupyter-archive==3.4.0 -- matplotlib==3.7.1 -- mdit-py-plugins==0.4.0 -- numpy==1.24.2 -- pandas==2.2.2 -- plotly==5.13.1 -- requests==2.28.2 -- scikit-image==0.19.3 -- scikit-learn==1.2.2 -- scipy==1.10.1 -- seaborn==0.12.2 -- statsmodels==0.14.0 -- tensorflow-cpu==2.12.1 -- sqlalchemy==2.0.16 -- mlxtend==0.23.0 -# Spring 2024 data 100 -- pytorch==2.1.2 -- cpuonly==2.0 -- transformers==4.37.1 -# Spring 2024 table demos -- lxml==5.1.0 -# Spring 2024 Econ 148 Packages -- geopandas==0.14.2 -- geopy==2.4.1 -- lifelines==0.27.8 -- pycountry==22.3.5 -- pip -- pip: - # - -r infra-requirements.txt - - ipywidgets==8.0.7 - # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) - # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 - - nbconvert[webpdf] - # - pyppeteer==2.0.0 - - pytest-notebook==0.8.1 - - gh-scoped-creds==4.1 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - - ydata-profiling==4.6.4 - - otter-grader==5.4.0 - - duckdb==0.10.1 - - duckdb_engine==0.11.2 - # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 - - jupytext==1.16.2 diff --git a/deployments/logodev/image/infra-requirements.txt b/deployments/logodev/image/infra-requirements.txt deleted file mode 100644 index b01c9d80d..000000000 --- a/deployments/logodev/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/logodev/image/postBuild b/deployments/logodev/image/postBuild deleted file mode 100644 index 9514de431..000000000 --- a/deployments/logodev/image/postBuild +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# installing chromium browser to enable webpdf conversion using nbconvert -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -npm install -g playwright -playwright install chromium From 5baf2e523ecf03a062f25ca2f13fe7b6bc54138d Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Aug 2024 10:13:03 -0700 Subject: [PATCH 2/5] adding hubploy testing --- .github/workflows/deploy-to-staging.yaml | 37 ++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/deploy-to-staging.yaml diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml new file mode 100644 index 000000000..d6e750723 --- /dev/null +++ b/.github/workflows/deploy-to-staging.yaml @@ -0,0 +1,37 @@ +name: Deploy images to hubs + +on: + push: + branches: + - staging + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v44 + with: + files_ignore: | + .github/** + images/** \ No newline at end of file From 2339589dcdbe46d17bc25e3a596596fd2a86d93f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 16 Aug 2024 09:50:16 -0700 Subject: [PATCH 3/5] placeholder commit --- .github/workflows/deploy-to-staging.yaml | 30 +++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index d6e750723..c041d210d 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -1,5 +1,5 @@ -name: Deploy images to hubs - +name: Deploy images to staging hubs +# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it on: push: branches: @@ -9,6 +9,12 @@ jobs: deploy: runs-on: ubuntu-latest steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout uses: actions/checkout@v4 @@ -28,10 +34,18 @@ jobs: pip install -r requirements.txt pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v44 + - name: Install Google Cloud SDK + uses: google-github-actions/setup-gcloud@v0.2.0 with: - files_ignore: | - .github/** - images/** \ No newline at end of file + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + + - name: Install SOPS + run: | + echo $SOPS_ACCOUNT_KEY > ${HOME}/repo/sops.key + echo 'export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/repo/sops.key' >> ${BASH_ENV} + mkdir -p ${HOME}/repo/bin + curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/repo/bin/sops + chmod 755 ${HOME}/repo/bin/sops + echo 'export PATH="${HOME}/repo/bin:${PATH}"' >> ${BASH_ENV} \ No newline at end of file From b84d3707b00b0ae6122fa606ebddbd35843741c8 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 10:28:27 -0700 Subject: [PATCH 4/5] initial commit to test gcloud github actions --- .github/workflows/deploy-to-staging.yaml | 22 +++++++++++++++------- deployments/logodev/hubploy.yaml | 5 ----- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index c041d210d..275cfcc46 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -1,6 +1,7 @@ name: Deploy images to staging hubs # use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it on: + workflow_dispatch: push: branches: - staging @@ -28,24 +29,31 @@ jobs: with: fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + - name: print out labels just for testing + run: echo "${{ steps.pr-labels.outputs.labels }}" + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + hubploy --help + hubploy deploy --help - - name: Install Google Cloud SDK - uses: google-github-actions/setup-gcloud@v0.2.0 + - name: Auth to gcloud + uses: google-github-actions/auth@v2 with: + credentials_json: ${{ secrets.GCP_SA_KEY }} project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true + + - name: Install Google Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + + - name: gcloud info + run: gcloud info - name: Install SOPS run: | - echo $SOPS_ACCOUNT_KEY > ${HOME}/repo/sops.key - echo 'export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/repo/sops.key' >> ${BASH_ENV} mkdir -p ${HOME}/repo/bin curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/repo/bin/sops chmod 755 ${HOME}/repo/bin/sops - echo 'export PATH="${HOME}/repo/bin:${PATH}"' >> ${BASH_ENV} \ No newline at end of file diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index 36e817577..fd2b36d2a 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -2,11 +2,6 @@ images: images: # temporary update - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:829f9eaececd - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json cluster: provider: gcloud From c8103463e858f8198a6f81cb6e4a83eb9db1c8d4 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 10:32:40 -0700 Subject: [PATCH 5/5] why is there a conflict --- deployments/logodev/image/apt.txt | 93 +++++++++++++++++++ deployments/logodev/image/environment.yml | 65 +++++++++++++ .../logodev/image/infra-requirements.txt | 29 ++++++ deployments/logodev/image/postBuild | 6 ++ 4 files changed, 193 insertions(+) create mode 100644 deployments/logodev/image/apt.txt create mode 100644 deployments/logodev/image/environment.yml create mode 100644 deployments/logodev/image/infra-requirements.txt create mode 100644 deployments/logodev/image/postBuild diff --git a/deployments/logodev/image/apt.txt b/deployments/logodev/image/apt.txt new file mode 100644 index 000000000..a6c4589ed --- /dev/null +++ b/deployments/logodev/image/apt.txt @@ -0,0 +1,93 @@ +# Some linux packages for basic terminal work, particularly +# oriented at users new to Unix/cmd line environments. + +# installing less as more just isn't enough +less + +# Basic unix tools +man +man-db +manpages-posix +manpages-dev +manpages-posix-dev + +# Download tools +curl +wget + +# Core text editors on a *nix box: vim +vim + +# A couple of CLI editors that are easier than vim +# micro # currently not working on 18.04 +nano +jed +jed-extra + +# powerful terminal-based file manager, better than the one in JLab +mc + +# for easily managing multiple repositories with one command (perl-doc +# is needed for its help pages to work) +mr +perl-doc + +# Regular build tools for compiling common stuff +build-essential +gfortran + +# Dependencies for nbconvert +texlive-xetex +texlive-fonts-recommended +texlive-plain-generic +# https://github.com/berkeley-dsep-infra/datahub/issues/3719 +texlive-lang-chinese +lmodern + +# Other useful document-related tools +pandoc +latexdiff + +# Some useful git utilities use basic Ruby +ruby + +# Other niceties for command-line work and life +ack # powerful grep-like tool +pydf # colorized disk usage +tmux +screen +htop +nnn # cmd line file manager +zsh +rsync +tig # console UI for git +multitail + +# For later, these are not available in 18.04 +#browsh # text-based web browser, occasionally handy +#dasel # json/yml/csv/etc data wrangling at the terminal +#fzf # fuzzy file finder + +## This section adds tools for desktop environment usage +dbus-x11 +xorg +xubuntu-icon-theme +xfce4 +xfce4-goodies +xclip +xsel +firefox +chromium-browser + +# GUI text editors +vim-gtk3 +gedit + +# Git clients and tools +git-gui +gitg +qgit +meld + +# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 +zip diff --git a/deployments/logodev/image/environment.yml b/deployments/logodev/image/environment.yml new file mode 100644 index 000000000..c4de2ee40 --- /dev/null +++ b/deployments/logodev/image/environment.yml @@ -0,0 +1,65 @@ +name: data100-FA23 + +channels: +- conda-forge +- pytorch + +dependencies: +- python==3.11.* +- git==2.39.1 +- jupyter-resource-usage==1.0.0 +- jupyterlab==4.0.11 +- jupyterlab-favorites==3.0.0 +- jupyterlab_server==2.23.0 +- jupyterlab_widgets==3.0.8 +- jupyter_server==2.7.0 +- nbgitpuller==1.2.1 +- notebook==7.0.7 +- folium==0.14.0 +- h5netcdf==1.0.2 +- ipywidgets==8.0.7 +- jupysql==0.8.0 +- jupyter-archive==3.4.0 +- matplotlib==3.7.1 +- mdit-py-plugins==0.4.0 +- numpy==1.24.2 +- pandas==2.2.2 +- plotly==5.13.1 +- requests==2.28.2 +- scikit-image==0.19.3 +- scikit-learn==1.2.2 +- scipy==1.10.1 +- seaborn==0.12.2 +- statsmodels==0.14.0 +- tensorflow-cpu==2.12.1 +- sqlalchemy==2.0.16 +- mlxtend==0.23.0 +# Spring 2024 data 100 +- pytorch==2.1.2 +- cpuonly==2.0 +- transformers==4.37.1 +# Spring 2024 table demos +- lxml==5.1.0 +# Spring 2024 Econ 148 Packages +- geopandas==0.14.2 +- geopy==2.4.1 +- lifelines==0.27.8 +- pycountry==22.3.5 +- pip +- pip: + # - -r infra-requirements.txt + - ipywidgets==8.0.7 + # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) + # - jupyter_collaboration==1.0.1 + - jupyterhub==4.1.6 + - nbconvert[webpdf] + # - pyppeteer==2.0.0 + - pytest-notebook==0.8.1 + - gh-scoped-creds==4.1 + - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + - ydata-profiling==4.6.4 + - otter-grader==5.4.0 + - duckdb==0.10.1 + - duckdb_engine==0.11.2 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/deployments/logodev/image/infra-requirements.txt b/deployments/logodev/image/infra-requirements.txt new file mode 100644 index 000000000..0fb0bd930 --- /dev/null +++ b/deployments/logodev/image/infra-requirements.txt @@ -0,0 +1,29 @@ +# WARNING: Original source at scripts/infra-packages/requirements.txt +# PLEASE DO NOT EDIT ELSEWHERE +# After editing scripts/infra-packages/requirements.txt, please run +# scripts/infra-packages/sync.bash. + +# This file pins versions of notebook related python packages we want +# across all hubs. This makes sure we don't need to upgrade them +# everwhere one by one. + +# FIXME: Freeze this to get exact versions of all dependencies +notebook==7.0.7 +jupyterlab==4.0.11 +nbgitpuller==1.2.1 +jupyter-resource-usage==1.0.1 +# Matches version in images/hub/Dockerfile +jupyterhub==4.1.6 +appmode==0.8.0 +ipywidgets==8.0.7 +jupyter-tree-download==1.0.1 +git-credential-helpers==0.2 +# Measure popularity of different packages in our hubs +# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda +git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling +# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 +RISE==5.7.1 +# syncthing for dropbox-like functionality +jupyter-syncthing-proxy==1.0.3 +# jupyter archival tool for easy user downloads +jupyter-archive==3.4.0 diff --git a/deployments/logodev/image/postBuild b/deployments/logodev/image/postBuild new file mode 100644 index 000000000..9514de431 --- /dev/null +++ b/deployments/logodev/image/postBuild @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +npm install -g playwright +playwright install chromium