diff --git a/.circleci/config.yml b/.circleci/config.yml index 35e38f2ec..4d91a6a84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -283,11 +283,11 @@ jobs: hubploy deploy --timeout 30m julia hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy logodev - command: | - hubploy deploy --timeout 30m logodev hub ${CIRCLE_BRANCH} - no_output_timeout: 30m +# - run: +# name: Deploy logodev +# command: | +# hubploy deploy --timeout 30m logodev hub ${CIRCLE_BRANCH} +# no_output_timeout: 30m - run: name: Deploy prob140 @@ -581,15 +581,15 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: logodev - name: logodev hub image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod +# - hubploy/build-image: +# deployment: logodev +# name: logodev hub image build +# # Filters can only be per-job? wtf +# filters: +# branches: +# ignore: +# - staging +# - prod - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -754,15 +754,15 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: logodev - name: logodev image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging +# - hubploy/build-image: +# deployment: logodev +# name: logodev image build +# push: true +# # Filters can only be per-job? wtf +# filters: +# branches: +# only: +# - staging - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -819,7 +819,7 @@ workflows: - eecs image build - ischool image build - julia hub image build - - logodev image build +# - logodev image build - publichealth image build - shiny image build - stat159 image build diff --git a/deployments/datahub/config/common.yaml b/deployments/datahub/config/common.yaml index bc77e9c64..54776aa56 100644 --- a/deployments/datahub/config/common.yaml +++ b/deployments/datahub/config/common.yaml @@ -106,6 +106,8 @@ jupyterhub: hub.jupyter.org/pool-name: core-pool-2024-05-08 singleuser: extraEnv: + GH_SCOPED_CREDS_CLIENT_ID: Iv23ct7Qx1mAotaIIYx9 + GH_SCOPED_CREDS_APP_URL: https://github.com/apps/berkeley-datahub-git-access # Unset NotebookApp from hub/values. Necessary for recent lab versions. JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp" nodeSelector: diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml index 757b532da..8086229e2 100644 --- a/deployments/datahub/images/default/environment.yml +++ b/deployments/datahub/images/default/environment.yml @@ -217,3 +217,10 @@ dependencies: # [DH-243] https://github.com/berkeley-dsep-infra/datahub/issues/5571, Legal Studies 123, Spring 24 - geopandas==0.14.3 - rtree==1.2.0 + + # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, ESPM 157 + - altair==5.3.0 + - ibis-framework[pandas]==9.2.0 + - leafmap==0.36.1 + - mystmd==1.3.0 + - jupyterlab-git==0.50.1 diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index 546a6a4f7..7760f94a1 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a02e39fbaedb path: image/ repo2docker: base_image: docker.io/library/buildpack-deps:jammy diff --git a/docs/admins/howto/index.rst b/docs/admins/howto/index.rst index 36f4cbedb..c6a0dd6ea 100644 --- a/docs/admins/howto/index.rst +++ b/docs/admins/howto/index.rst @@ -11,9 +11,11 @@ Common Administrator Tasks core-pool new-hub rebuild-hub-image + new-image new-packages course-config calendar-scaler prometheus-grafana remove-users-orm delete-hub + clusterswitch.md diff --git a/docs/admins/howto/new-image.rst b/docs/admins/howto/new-image.rst new file mode 100644 index 000000000..22b5cb549 --- /dev/null +++ b/docs/admins/howto/new-image.rst @@ -0,0 +1,97 @@ +.. _howto/new-packages: + +================================ +Creating a new single user image +================================ + +When deploying a new hub, or moving from a shared single user server image, +you might need to create a new image for users. We use +`repo2docker `_ to do this. + +There are two approaches to creating a repo2docker image: +1. Use a repo2docker-style image `template `_ (environment.yaml, etc) +2. Use a `Dockerfile `_ (useful for larger/more complex images) + +Generally, we prefer to use the former approach, unless we need to install +specific packages or utilities outside of python/apt as ``root``. If that is +the case, only a :file:`Dockerfile` format will work. + +Of course, as always create a feature branch for your changes, and submit a +PR when done. + +Find a hub to use as a template +=============================== + +Browse through our :file:`deployments/` directory to find a hub that is similar to +the one you are trying to create. This will give you a good starting point. + +Create the :file:`image/` directory for your new hub +==================================================== + +Create a new directory under :file:`deployments/` with the name of your hub. This +directory will contain the files that will be used to create the image. + +Then, copy the contents (and any subdirectories) of the source :file:`image/` +directory in to the new directory. + +Modify :file:`hubploy.yaml` for the hub +======================================= + +In the deployment's :file:`hubploy.yaml` file, add or modify the ``name``, ``path`` and +``base_image`` fields to configure the image build and where it's stored in the +Google Artifcat Registry. + +``name`` should contain the path to the image in the Google Artifact Registry and the name of the image. +``path`` points to the directory containing the image configuration (typically :file::`image/`). +``base_image`` is the base Docker image to use for the image build. + +For example, :file:`hubploy.yaml` for the data100 image looks like this: + +.. code:: yaml + + images: + images: + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data100-user-image + path: image/ + repo2docker: + base_image: docker.io/library/buildpack-deps:jammy + registry: + provider: gcloud + gcloud: + project: ucb-datahub-2018 + service_key: gcr-key.json + + cluster: + provider: gcloud + gcloud: + project: ucb-datahub-2018 + service_key: gke-key.json + cluster: spring-2024 + zone: us-central1 + +Modify the image configuration as necessary +=========================================== + +This step is straightforward: edit/modify/delete/add any files in the :file:`image/` +directory to configure the image as needed. + +Update CI/CD configuration +========================== + +Next, ensure that this image will be built and deployed by updating the +:file:`.circleci/config.yml` file in the root of the repository. Add new steps +under the ``jobs/deploy:``, ``workflows/test-build-images:`` and ``workflows/deploy:`` +stanzas. + +Submitting a pull request +========================= + +Familiarize yourself with `pull requests `_ and `repo2docker `_ , and create a fork of the `datahub staging branch `_. + +#. Set up your git/dev environment by `following the instructions here `_. +#. Create a new branch for this PR. +#. Test the changes locally using ``repo2docker``, then submit a PR to ``staging``. + * To use ``repo2docker``, you have to point it at the correct image directory. For example, to build the data100 image, you would run ``repo2docker deployments/data100/image`` from the base datahub directory. +#. Commit and push your changes to your fork of the datahub repo, and create a new pull request at ``__. +#. Once the PR is merged to staging and the new image is built and pushed to Artifact Registry, you can test it out on :code:`-staging.datahub.berkeley.edu`. +#. Changes are only deployed to prod once the relevant CI job is completed. See ``__ to view CircleCI job statuses. diff --git a/docs/conf.py b/docs/conf.py index 0c21b6a9f..397734f25 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -68,7 +68,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = None +pygments_style = 'sphinx' # -- Options for HTML output ------------------------------------------------- diff --git a/docs/users/index.rst b/docs/users/index.rst index 9c0b36fa9..8215c1fdd 100644 --- a/docs/users/index.rst +++ b/docs/users/index.rst @@ -8,4 +8,3 @@ Using DataHub private-repo hubs authentication - storage-retention diff --git a/requirements.txt b/requirements.txt index ba67b08a4..da3a1ba42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,6 @@ chartpress jupyter-repo2docker==2024.3.0 myst-parser chardet -# requests==2.31.0 -requests==2.32.3 -urllib3==2.2.2 +requests==2.31.0 +# requests==2.32.3 +urllib3<2.0.0