From fd2df8e366d3b19a8d72024c5e33444f14066cc1 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Wed, 24 Jul 2024 18:22:03 -0700 Subject: [PATCH 1/8] Add ESPM 157 sepcific packages and enable gh-scoped-creds config --- deployments/datahub/config/common.yaml | 2 ++ deployments/datahub/images/default/environment.yml | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/deployments/datahub/config/common.yaml b/deployments/datahub/config/common.yaml index bc77e9c64..54776aa56 100644 --- a/deployments/datahub/config/common.yaml +++ b/deployments/datahub/config/common.yaml @@ -106,6 +106,8 @@ jupyterhub: hub.jupyter.org/pool-name: core-pool-2024-05-08 singleuser: extraEnv: + GH_SCOPED_CREDS_CLIENT_ID: Iv23ct7Qx1mAotaIIYx9 + GH_SCOPED_CREDS_APP_URL: https://github.com/apps/berkeley-datahub-git-access # Unset NotebookApp from hub/values. Necessary for recent lab versions. JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp" nodeSelector: diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml index 757b532da..f0c58ebba 100644 --- a/deployments/datahub/images/default/environment.yml +++ b/deployments/datahub/images/default/environment.yml @@ -217,3 +217,10 @@ dependencies: # [DH-243] https://github.com/berkeley-dsep-infra/datahub/issues/5571, Legal Studies 123, Spring 24 - geopandas==0.14.3 - rtree==1.2.0 + + # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, + - altair==5.3.0 + - ibis-framework==9.2.0 + - leafmap==0.36.1 + - mystmd==1.3.0 + - jupyterlab-git==0.50.1 From d6bc17ec9faadfdda595fe31d8bcc47ae9c09c59 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 25 Jul 2024 11:35:54 -0700 Subject: [PATCH 2/8] new user image doc --- docs/admins/howto/index.rst | 1 + docs/admins/howto/new-image.rst | 96 +++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 docs/admins/howto/new-image.rst diff --git a/docs/admins/howto/index.rst b/docs/admins/howto/index.rst index 36f4cbedb..b8eece414 100644 --- a/docs/admins/howto/index.rst +++ b/docs/admins/howto/index.rst @@ -11,6 +11,7 @@ Common Administrator Tasks core-pool new-hub rebuild-hub-image + new-image new-packages course-config calendar-scaler diff --git a/docs/admins/howto/new-image.rst b/docs/admins/howto/new-image.rst new file mode 100644 index 000000000..fab8375e3 --- /dev/null +++ b/docs/admins/howto/new-image.rst @@ -0,0 +1,96 @@ +.. _howto/new-packages: + +================================ +Creating a new single user image +================================ + +When deploying a new hub, or moving from a shared single user server image, +you might need to create a new image for users. We use +`repo2docker `_ to do this. + +There are two approaches to creating a repo2docker image: +1. Use a repo2docker-style image `template `_ (environment.yaml, etc) +2. Use a `Dockerfile `_ (useful for larger/more complex images) + +Generally, we prefer to use the former approach, unless we need to install +specific packages or utilities outside of python/apt as `root`. If that is +the case, only a `Dockerfile` format will work. + +Of course, as always create a feature branch for your changes, and submit a +PR when done. + +Find a hub to use as a template +=============================== + +Browse through our `deployments/` directory to find a hub that is similar to +the one you are trying to create. This will give you a good starting point. + +Create the :file:`image/` directory for your new hub +==================================================== + +Create a new directory under `deployments/` with the name of your hub. This +directory will contain the files that will be used to create the image. + +Then, copy the contents (and any subdirectories) of the source `image/` +directory in to the new directory. + +Modify :file:`hubploy.yaml` for the hub +======================================= + +In the deployment's :file:`hubploy.yaml` file, add or modify the `name`, `path` and +`base_image` fields to configure the image build and where it's stored in the +Google Artifcat Registry. + +`name` should contain the path to the image in the Google Artifact Registry and the name of the image. +`path` points to the directory containing the image configuration (typically :file::`image/`). +`base_image` is the base Docker image to use for the image build. + +For example, the `hubploy.yaml` for the data100 image looks like this: + + .. code-block:: yaml + images: + images: + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data100-user-image + path: image/ + repo2docker: + base_image: docker.io/library/buildpack-deps:jammy + registry: + provider: gcloud + gcloud: + project: ucb-datahub-2018 + service_key: gcr-key.json + + cluster: + provider: gcloud + gcloud: + project: ucb-datahub-2018 + service_key: gke-key.json + cluster: spring-2024 + zone: us-central1 + +Modify the image configuration as necessary +=========================================== + +This step is straightforward: edit/modify/delete/add any files in the `image/` +directory to configure the image as needed. + +Update CI/CD configuration +========================== + +Next, ensure that this image will be built and deployed by updating the +:file:`.circleci/config.yml` file in the root of the repository. Add new steps +under the `jobs/deploy:`, `workflows/test-build-images:` and `workflows/deploy:` +stanzas. + +Submitting a pull request +========================= + +Familiarize yourself with `pull requests `_ and `repo2docker `_ , and create a fork of the `datahub staging branch `_. + +#. Set up your git/dev environment by `following the instructions here `_. +#. Create a new branch for this PR. +#. Test the changes locally using `repo2docker`, then submit a PR to ``staging``. + * To use ``repo2docker``, you have to point it at the correct image directory. For example, to build the data100 image, you would run ``repo2docker deployments/data100/image`` from the base datahub directory. +#. Commit and push your changes to your fork of the datahub repo, and create a new pull request at ``__. +#. Once the PR is merged to staging and the new image is built and pushed to Artifact Registry, you can test it out on :code:`-staging.datahub.berkeley.edu`. +#. Changes are only deployed to prod once the relevant CI job is completed. See ``__ to view CircleCI job statuses. From fa70180d999f80776f8c8bfdaf32953baab5a7ab Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 25 Jul 2024 11:46:21 -0700 Subject: [PATCH 3/8] stupid requests --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ba67b08a4..eb8d858c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,6 @@ chartpress jupyter-repo2docker==2024.3.0 myst-parser chardet -# requests==2.31.0 -requests==2.32.3 +requests==2.31.0 +# requests==2.32.3 urllib3==2.2.2 From 48f9991b3b94823c0ca5f892d58255733edb1848 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 25 Jul 2024 11:53:26 -0700 Subject: [PATCH 4/8] grrr --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eb8d858c7..da3a1ba42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ myst-parser chardet requests==2.31.0 # requests==2.32.3 -urllib3==2.2.2 +urllib3<2.0.0 From 30a0a867217eb3485849a6e491c4fc0a25f8260f Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Thu, 25 Jul 2024 13:11:10 -0700 Subject: [PATCH 5/8] Replacing ibis-framework package with ibis-pandas --- deployments/datahub/images/default/environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml index f0c58ebba..bf0a37de9 100644 --- a/deployments/datahub/images/default/environment.yml +++ b/deployments/datahub/images/default/environment.yml @@ -218,9 +218,9 @@ dependencies: - geopandas==0.14.3 - rtree==1.2.0 - # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, + # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, ESPM 157 - altair==5.3.0 - - ibis-framework==9.2.0 + - ibis-pandas==9.2.0 - leafmap==0.36.1 - mystmd==1.3.0 - jupyterlab-git==0.50.1 From 3841e8a40e83797037fbc311aaebe01f2060c449 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 25 Jul 2024 13:41:42 -0700 Subject: [PATCH 6/8] fixing doc --- docs/admins/howto/index.rst | 1 + docs/admins/howto/new-image.rst | 31 ++++++++++++++++--------------- docs/conf.py | 2 +- docs/users/index.rst | 1 - 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/admins/howto/index.rst b/docs/admins/howto/index.rst index b8eece414..c6a0dd6ea 100644 --- a/docs/admins/howto/index.rst +++ b/docs/admins/howto/index.rst @@ -18,3 +18,4 @@ Common Administrator Tasks prometheus-grafana remove-users-orm delete-hub + clusterswitch.md diff --git a/docs/admins/howto/new-image.rst b/docs/admins/howto/new-image.rst index fab8375e3..22b5cb549 100644 --- a/docs/admins/howto/new-image.rst +++ b/docs/admins/howto/new-image.rst @@ -13,8 +13,8 @@ There are two approaches to creating a repo2docker image: 2. Use a `Dockerfile `_ (useful for larger/more complex images) Generally, we prefer to use the former approach, unless we need to install -specific packages or utilities outside of python/apt as `root`. If that is -the case, only a `Dockerfile` format will work. +specific packages or utilities outside of python/apt as ``root``. If that is +the case, only a :file:`Dockerfile` format will work. Of course, as always create a feature branch for your changes, and submit a PR when done. @@ -22,32 +22,33 @@ PR when done. Find a hub to use as a template =============================== -Browse through our `deployments/` directory to find a hub that is similar to +Browse through our :file:`deployments/` directory to find a hub that is similar to the one you are trying to create. This will give you a good starting point. Create the :file:`image/` directory for your new hub ==================================================== -Create a new directory under `deployments/` with the name of your hub. This +Create a new directory under :file:`deployments/` with the name of your hub. This directory will contain the files that will be used to create the image. -Then, copy the contents (and any subdirectories) of the source `image/` +Then, copy the contents (and any subdirectories) of the source :file:`image/` directory in to the new directory. Modify :file:`hubploy.yaml` for the hub ======================================= -In the deployment's :file:`hubploy.yaml` file, add or modify the `name`, `path` and -`base_image` fields to configure the image build and where it's stored in the +In the deployment's :file:`hubploy.yaml` file, add or modify the ``name``, ``path`` and +``base_image`` fields to configure the image build and where it's stored in the Google Artifcat Registry. -`name` should contain the path to the image in the Google Artifact Registry and the name of the image. -`path` points to the directory containing the image configuration (typically :file::`image/`). -`base_image` is the base Docker image to use for the image build. +``name`` should contain the path to the image in the Google Artifact Registry and the name of the image. +``path`` points to the directory containing the image configuration (typically :file::`image/`). +``base_image`` is the base Docker image to use for the image build. -For example, the `hubploy.yaml` for the data100 image looks like this: +For example, :file:`hubploy.yaml` for the data100 image looks like this: + +.. code:: yaml - .. code-block:: yaml images: images: - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data100-user-image @@ -71,7 +72,7 @@ For example, the `hubploy.yaml` for the data100 image looks like this: Modify the image configuration as necessary =========================================== -This step is straightforward: edit/modify/delete/add any files in the `image/` +This step is straightforward: edit/modify/delete/add any files in the :file:`image/` directory to configure the image as needed. Update CI/CD configuration @@ -79,7 +80,7 @@ Update CI/CD configuration Next, ensure that this image will be built and deployed by updating the :file:`.circleci/config.yml` file in the root of the repository. Add new steps -under the `jobs/deploy:`, `workflows/test-build-images:` and `workflows/deploy:` +under the ``jobs/deploy:``, ``workflows/test-build-images:`` and ``workflows/deploy:`` stanzas. Submitting a pull request @@ -89,7 +90,7 @@ Familiarize yourself with `pull requests `_. #. Create a new branch for this PR. -#. Test the changes locally using `repo2docker`, then submit a PR to ``staging``. +#. Test the changes locally using ``repo2docker``, then submit a PR to ``staging``. * To use ``repo2docker``, you have to point it at the correct image directory. For example, to build the data100 image, you would run ``repo2docker deployments/data100/image`` from the base datahub directory. #. Commit and push your changes to your fork of the datahub repo, and create a new pull request at ``__. #. Once the PR is merged to staging and the new image is built and pushed to Artifact Registry, you can test it out on :code:`-staging.datahub.berkeley.edu`. diff --git a/docs/conf.py b/docs/conf.py index 0c21b6a9f..397734f25 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -68,7 +68,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = None +pygments_style = 'sphinx' # -- Options for HTML output ------------------------------------------------- diff --git a/docs/users/index.rst b/docs/users/index.rst index 9c0b36fa9..8215c1fdd 100644 --- a/docs/users/index.rst +++ b/docs/users/index.rst @@ -8,4 +8,3 @@ Using DataHub private-repo hubs authentication - storage-retention From c9191d1bded676c33ac16843c85fb1ddcd9afc29 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Thu, 25 Jul 2024 14:50:25 -0700 Subject: [PATCH 7/8] under pip block, install via pip --- deployments/datahub/images/default/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml index bf0a37de9..8086229e2 100644 --- a/deployments/datahub/images/default/environment.yml +++ b/deployments/datahub/images/default/environment.yml @@ -220,7 +220,7 @@ dependencies: # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, ESPM 157 - altair==5.3.0 - - ibis-pandas==9.2.0 + - ibis-framework[pandas]==9.2.0 - leafmap==0.36.1 - mystmd==1.3.0 - jupyterlab-git==0.50.1 From db99fba910d08ef0c8e60aab0ff934cee353c86f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 29 Jul 2024 13:15:49 -0700 Subject: [PATCH 8/8] testing for DH-301 --- .circleci/config.yml | 48 ++++++++++++++++---------------- deployments/logodev/hubploy.yaml | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 35e38f2ec..4d91a6a84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -283,11 +283,11 @@ jobs: hubploy deploy --timeout 30m julia hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy logodev - command: | - hubploy deploy --timeout 30m logodev hub ${CIRCLE_BRANCH} - no_output_timeout: 30m +# - run: +# name: Deploy logodev +# command: | +# hubploy deploy --timeout 30m logodev hub ${CIRCLE_BRANCH} +# no_output_timeout: 30m - run: name: Deploy prob140 @@ -581,15 +581,15 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: logodev - name: logodev hub image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod +# - hubploy/build-image: +# deployment: logodev +# name: logodev hub image build +# # Filters can only be per-job? wtf +# filters: +# branches: +# ignore: +# - staging +# - prod - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -754,15 +754,15 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: logodev - name: logodev image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging +# - hubploy/build-image: +# deployment: logodev +# name: logodev image build +# push: true +# # Filters can only be per-job? wtf +# filters: +# branches: +# only: +# - staging - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -819,7 +819,7 @@ workflows: - eecs image build - ischool image build - julia hub image build - - logodev image build +# - logodev image build - publichealth image build - shiny image build - stat159 image build diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index 546a6a4f7..7760f94a1 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a02e39fbaedb path: image/ repo2docker: base_image: docker.io/library/buildpack-deps:jammy