From bab61d92210fb08e896fa21b48ff7b1fb092dce5 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 8 Oct 2024 16:21:58 -0700 Subject: [PATCH 01/15] configs for pre-commit checks --- .codespellrc | 3 ++ .flake8 | 11 ++++++ .pre-commit-config.yaml | 75 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 .codespellrc create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 000000000..299070081 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,3 @@ +[codespell] +skip = '.git,*.svg,*.secret.*,*.key,./vendor/grafana/dashboards/*' +ignore-words-list = 'demog' diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..fb5e07587 --- /dev/null +++ b/.flake8 @@ -0,0 +1,11 @@ +[flake8] +# Ignore style and complexity +# E: style errors +# W: style warnings +# C: complexity +# E402: module level import not at top of file +# I100: Import statements are in the wrong order +# I101: Imported names are in the wrong order. Should be +# F841: local variable ... is assigned to but never used +ignore = E, C, W, I100, I101, D400, F841 +builtins = c diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..65bea6d47 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,75 @@ +# pre-commit is a tool to perform a predefined set of tasks manually and/or +# automatically before git commits are made. +# +# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level +# +# Common tasks +# +# - Run on all files: pre-commit run --all-files +# - Register git hooks: pre-commit install --install-hooks +# +repos: + # Autoformat: Python code + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.6.9 + hooks: + # Run the linter. + - id: ruff + types_or: [ python, pyi ] + args: [ --fix ] + # Run the formatter. + - id: ruff-format + types_or: [ python, pyi ] + + # Autoformat: Python code, syntax patterns are modernized + - repo: https://github.com/asottile/pyupgrade + rev: v3.17.0 + hooks: + - id: pyupgrade + args: + - --py36-plus + + # Autoformat: Python code + - repo: https://github.com/pycqa/isort + rev: "5.13.2" + hooks: + - id: isort + + # Autoformat: Python code + - repo: https://github.com/psf/black + rev: "24.10.0" + hooks: + - id: black + # This is a `.py` file but is encrypted with sops + + # Lint: Python code + - repo: https://github.com/pycqa/flake8 + rev: "7.1.1" + hooks: + - id: flake8 + # This is a `.py` file but is encrypted with sops + exclude: deployer/commands/generate/cryptnono_config/enc-blocklist-generator.secret.py + + # Prevent unencrypted files from being committed + - repo: https://github.com/yuvipanda/pre-commit-hook-ensure-sops + rev: v1.1 + hooks: + - id: sops-encryption + # Add files here if they contain the word 'secret' but should not be encrypted + exclude: secrets\.md|helm-charts/support/templates/prometheus-ingres-auth/secret\.yaml|helm-charts/basehub/templates/dex/secret\.yaml|helm-charts/basehub/templates/static/secret\.yaml|config/clusters/templates/common/support\.secret\.values\.yaml|helm-charts/basehub/templates/ingress-auth/secret\.yaml + + # Prevent known typos from being committed + - repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + args: + # Autofix known typos + - --write-changes + additional_dependencies: + - tomli + +# pre-commit.ci config reference: https://pre-commit.ci/#configuration +#ci: +# autoupdate_schedule: monthly From a04f4064aea23a48df9426eb3cd10cbc45db29d4 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 14 Oct 2024 14:35:56 -0700 Subject: [PATCH 02/15] final tweaking of configs for a (relatively) sane pre-commit workflow --- .codespellrc | 3 --- .flake8 | 1 - .pre-commit-config.yaml | 18 ++++++++++++++++++ pyproject.toml | 10 ++++++++++ 4 files changed, 28 insertions(+), 4 deletions(-) delete mode 100644 .codespellrc create mode 100644 pyproject.toml diff --git a/.codespellrc b/.codespellrc deleted file mode 100644 index 299070081..000000000 --- a/.codespellrc +++ /dev/null @@ -1,3 +0,0 @@ -[codespell] -skip = '.git,*.svg,*.secret.*,*.key,./vendor/grafana/dashboards/*' -ignore-words-list = 'demog' diff --git a/.flake8 b/.flake8 index fb5e07587..9419e9369 100644 --- a/.flake8 +++ b/.flake8 @@ -8,4 +8,3 @@ # I101: Imported names are in the wrong order. Should be # F841: local variable ... is assigned to but never used ignore = E, C, W, I100, I101, D400, F841 -builtins = c diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65bea6d47..f1f7a5f9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -70,6 +70,24 @@ repos: additional_dependencies: - tomli + # misc stuff + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available + hooks: + # Autoformat: Makes sure files end in a newline and only a newline. + - id: end-of-file-fixer + + # Autoformat: Sorts entries in requirements.txt. + - id: requirements-txt-fixer + + # Lint: Check for files with names that would conflict on a + # case-insensitive filesystem like MacOS HFS+ or Windows FAT. + - id: check-case-conflict + + # Lint: Checks that non-binary executables have a proper shebang. + - id: check-executables-have-shebangs + # pre-commit.ci config reference: https://pre-commit.ci/#configuration #ci: # autoupdate_schedule: monthly diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..7fbb59e8f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +[tool.black] +# target-version should be all supported versions +target-version = ['py37', 'py38', 'py39', 'py310'] + +[tool.isort] +profile = "black" + +[tool.codespell] +skip = '.git,*.pdf,*.svg,vendor/grafana/dashboards/*,deployments/*/secrets/*' +ignore-words-list = 'demog,wasn' From cd99ebff794df194e03a20e311697f2d4e2ebdcf Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 14 Oct 2024 14:41:30 -0700 Subject: [PATCH 03/15] remove some 2i2c specific stuff --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1f7a5f9a..2cb43fce2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,7 +57,7 @@ repos: hooks: - id: sops-encryption # Add files here if they contain the word 'secret' but should not be encrypted - exclude: secrets\.md|helm-charts/support/templates/prometheus-ingres-auth/secret\.yaml|helm-charts/basehub/templates/dex/secret\.yaml|helm-charts/basehub/templates/static/secret\.yaml|config/clusters/templates/common/support\.secret\.values\.yaml|helm-charts/basehub/templates/ingress-auth/secret\.yaml + # exclude: # Prevent known typos from being committed - repo: https://github.com/codespell-project/codespell From 8263f2e83949890a561892a59442e8ab26ee31f1 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 15 Oct 2024 11:47:35 -0700 Subject: [PATCH 04/15] add py3.11 to black config --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7fbb59e8f..eea6392a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] # target-version should be all supported versions -target-version = ['py37', 'py38', 'py39', 'py310'] +target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] [tool.isort] profile = "black" From c05911beb83c84b4c1c2e976007053e6a8a6a38f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 15 Oct 2024 12:57:29 -0700 Subject: [PATCH 05/15] move yaml linter to pre-commit and remove GHA --- .github/workflows/yaml-lint.yaml | 15 --------------- .pre-commit-config.yaml | 7 +++++++ 2 files changed, 7 insertions(+), 15 deletions(-) delete mode 100644 .github/workflows/yaml-lint.yaml diff --git a/.github/workflows/yaml-lint.yaml b/.github/workflows/yaml-lint.yaml deleted file mode 100644 index 787d5e3fa..000000000 --- a/.github/workflows/yaml-lint.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: "Yaml lint" -on: - - pull_request # yamllint disable-line rule:truthy - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install yamllint - run: pip install yamllint==1.35.1 - - - name: Lint YAML files - run: yamllint --no-warnings . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2cb43fce2..4d50d27f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,13 @@ # - Register git hooks: pre-commit install --install-hooks # repos: + # yaml linter + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.35.1 + hooks: + - id: yamllint + args: [--no-warnings, -c=./.yamllint.yaml] + # Autoformat: Python code - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. From 9432c52413c28cb83b0521e94e2d9816e03612b6 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 17 Oct 2024 09:43:01 -0700 Subject: [PATCH 06/15] add TODO to eventually remove the args in yamllint that skips warnings --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d50d27f8..a2e41300e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,7 @@ repos: rev: v1.35.1 hooks: - id: yamllint + # TODO: remove --no-warnings eventually args: [--no-warnings, -c=./.yamllint.yaml] # Autoformat: Python code From a05f4efd8e759df183ad67fa6f5da83c82b23803 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 17 Oct 2024 16:13:32 -0700 Subject: [PATCH 07/15] minor tweaks in the config --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2e41300e..c0d63cb27 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,8 +56,8 @@ repos: rev: "7.1.1" hooks: - id: flake8 - # This is a `.py` file but is encrypted with sops - exclude: deployer/commands/generate/cryptnono_config/enc-blocklist-generator.secret.py + # In case a `.py` file but is encrypted with sops + # exclude: # Prevent unencrypted files from being committed - repo: https://github.com/yuvipanda/pre-commit-hook-ensure-sops @@ -78,7 +78,7 @@ repos: additional_dependencies: - tomli - # misc stuff + # pre-commit.com provided hooks - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available From 37b7d3cd9472fbdda1eea90d17555a7b4467fcfc Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 18 Oct 2024 11:02:28 -0700 Subject: [PATCH 08/15] adding dev reqs --- dev-requirements.txt | 13 +++++++++++++ requirements.txt | 1 - 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 dev-requirements.txt diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 000000000..fe0997139 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,13 @@ +# allow for base image specification in hubploy.yaml +git+https://github.com/berkeley-dsep-infra/hubploy.git +pygithub +pyrsistent==0.19.3 +attrs==23.1.0 +chartpress +jupyter-repo2docker==2024.07.0 +myst-parser +chardet +niquests==3.7.2 +urllib3<2.0.0 +yamllint==1.35.1 +pre-commit==4.0.1 diff --git a/requirements.txt b/requirements.txt index 88193e5d1..cf410a6e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,3 @@ chardet niquests==3.7.2 urllib3<2.0.0 yamllint==1.35.1 -pre-commit==4.0.1 From d8305841fef03f9d6337db0b1218ebc312a6f175 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 18 Oct 2024 11:03:46 -0700 Subject: [PATCH 09/15] adding dev reqs to labeler --- .github/labeler.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/labeler.yml b/.github/labeler.yml index a74875c95..4fa33334b 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -16,6 +16,7 @@ configuration: - '**/apt.txt' - '**/infra-requirements.txt' - '**/requirements.in' + - '**/dev-requirements.txt' - '**/requirements.txt' - '**/runtime.txt' - '**/*.json' From 38ab80927115b352b360cf4f175e48feb68dd046 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 18 Oct 2024 11:15:39 -0700 Subject: [PATCH 10/15] remove old comment from dev reqs --- dev-requirements.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index fe0997139..f6ab4ebff 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,13 +1,12 @@ -# allow for base image specification in hubploy.yaml -git+https://github.com/berkeley-dsep-infra/hubploy.git -pygithub -pyrsistent==0.19.3 attrs==23.1.0 +chardet chartpress +git+https://github.com/berkeley-dsep-infra/hubploy.git jupyter-repo2docker==2024.07.0 myst-parser -chardet niquests==3.7.2 +pre-commit==4.0.1 +pygithub +pyrsistent==0.19.3 urllib3<2.0.0 yamllint==1.35.1 -pre-commit==4.0.1 From 618d172c552aa210c37f1ca3dff01b7d9c306345 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 18 Oct 2024 11:17:43 -0700 Subject: [PATCH 11/15] massive PRs are my fav thing evar --- .github/ISSUE_TEMPLATE/featurerequest.md | 1 - .github/ISSUE_TEMPLATE/higher-resources.md | 1 - .github/ISSUE_TEMPLATE/performance_issue.yml | 4 +-- .github/ISSUE_TEMPLATE/resourcescheduler.yml | 4 +-- deployments/a11y/secrets/gke-key.json | 2 +- deployments/astro/secrets/gke-key.json | 2 +- deployments/biology/secrets/gke-key.json | 2 +- deployments/cee/secrets/gke-key.json | 2 +- deployments/data100/secrets/gke-key.json | 2 +- deployments/data101/secrets/gke-key.json | 2 +- deployments/data102/secrets/gke-key.json | 2 +- deployments/datahub/secrets/gke-key.json | 2 +- deployments/dev/secrets/gke-key.json | 2 +- deployments/dlab/secrets/gke-key.json | 2 +- deployments/edx/config/common.yaml | 2 -- deployments/edx/hubploy.yaml | 2 -- .../edx/secrets/gcloud-service-key.json | 2 +- deployments/eecs/config/common.yaml | 2 +- deployments/eecs/secrets/gke-key.json | 2 +- deployments/gradebook/secrets/gke-key.json | 2 +- deployments/highschool/secrets/gke-key.json | 2 +- deployments/ischool/secrets/gke-key.json | 2 +- deployments/julia/secrets/gke-key.json | 2 +- deployments/logodev/secrets/gke-key.json | 2 +- deployments/nature/secrets/gke-key.json | 2 +- deployments/prob140/secrets/gke-key.json | 2 +- deployments/publichealth/secrets/gke-key.json | 2 +- deployments/r/config/common.yaml | 1 - deployments/r/secrets/gke-key.json | 2 +- deployments/shiny/secrets/gke-key.json | 2 +- deployments/stat159/secrets/gke-key.json | 2 +- deployments/stat20/secrets/gke-key.json | 2 +- deployments/ugr01/secrets/gke-key.json | 2 +- deployments/workshop/secrets/gke-key.json | 2 +- docs/admins/cluster-config.qmd | 2 +- docs/admins/howto/calendar-scaler.qmd | 2 +- docs/admins/howto/course-config.qmd | 6 ++-- .../managing-multiple-user-image-repos.qmd | 4 +-- docs/admins/howto/new-hub.qmd | 4 +-- docs/admins/howto/new-packages.qmd | 2 +- docs/admins/howto/rebuild-hub-image.qmd | 2 +- docs/admins/howto/rebuild-postgres-image.qmd | 4 +-- docs/datahub.svg | 2 +- .../2017-02-24-autoscaler-incident.qmd | 2 -- .../incidents/2017-03-20-too-many-volumes.qmd | 4 +-- .../2017-03-23-kernel-deaths-incident.qmd | 2 +- .../2019-05-01-service-account-leak.qmd | 2 +- ...20-package-dependency-upgrade-incident.qmd | 2 +- docs/incidents/2024-core-node-incidents.qmd | 2 +- docs/policy/create_policy.qmd | 2 +- docs/policy/policy_deploy_mainhubs.qmd | 2 +- docs/policy/principles.qmd | 2 +- docs/users/hubs/stat159.qmd | 2 +- .../scaler/__main__.py | 1 - .../scaler/calendar.py | 2 +- .../node-placeholder-scaler/scaler/scaler.py | 14 ++++---- node-placeholder/templates/configmap.yaml | 2 +- node-placeholder/templates/priorityclass.yaml | 4 +-- node-placeholder/templates/rbac.yaml | 2 +- requirements.txt | 8 ++--- scripts/delete-unused-users.py | 6 ++-- scripts/git-pre-cloner.py | 2 +- scripts/infra-packages/requirements.txt | 34 +++++++++---------- scripts/list-packages.py | 7 ++-- scripts/post-grafana-annotation.py | 3 +- scripts/rsync-active-users.py | 18 +++++----- scripts/user-image-management/README.md | 4 +-- .../manage-image-repos.py | 2 +- vendor/google/ugr/README.md | 4 +-- .../Anomalous activities-1714066292129.json | 2 +- ...loud Storage Monitoring-1714066317471.json | 2 +- .../general/Cluster Health-1714066323007.json | 2 +- .../Cluster Information2-1714066330142.json | 2 +- ...ore pods resource usage-1714066335185.json | 2 +- ...P Requests (nginx) Copy-1714066338895.json | 2 +- .../general/Hub Metrics-1714066344166.json | 2 +- .../JupyterHub Dashboard2-1714066348840.json | 2 +- ...GINX Ingress controller-1714066353536.json | 2 +- ...thon Popularity Contest-1714066357213.json | 2 +- .../Ryan's Draft Panels-1714066360784.json | 2 +- .../Storage I_O Statistics-1714066365230.json | 2 +- ...x based request metrics-1714066371135.json | 2 +- .../Cluster Information-1714066576091.json | 2 +- .../Global Usage Dashboard-1714066534688.json | 2 +- .../JupyterHub Dashboard-1714066548245.json | 2 +- ...and Support Information-1714066550552.json | 2 +- .../Usage Report-1714066552403.json | 2 +- ...r Diagnostics Dashboard-1714066554584.json | 2 +- 88 files changed, 132 insertions(+), 138 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/featurerequest.md b/.github/ISSUE_TEMPLATE/featurerequest.md index a9d13ba87..d091e8238 100644 --- a/.github/ISSUE_TEMPLATE/featurerequest.md +++ b/.github/ISSUE_TEMPLATE/featurerequest.md @@ -30,4 +30,3 @@ assignees: # Tasks to complete - diff --git a/.github/ISSUE_TEMPLATE/higher-resources.md b/.github/ISSUE_TEMPLATE/higher-resources.md index ed8be0e1b..01c2039fa 100644 --- a/.github/ISSUE_TEMPLATE/higher-resources.md +++ b/.github/ISSUE_TEMPLATE/higher-resources.md @@ -45,4 +45,3 @@ This will help us with the prioritization of this request. This will help us scale down resources when the RAM increase is no longer required **Any additional information we should know about?** - diff --git a/.github/ISSUE_TEMPLATE/performance_issue.yml b/.github/ISSUE_TEMPLATE/performance_issue.yml index 1555d549a..5e08e92b1 100644 --- a/.github/ISSUE_TEMPLATE/performance_issue.yml +++ b/.github/ISSUE_TEMPLATE/performance_issue.yml @@ -9,7 +9,7 @@ body: - type: markdown attributes: value: | - Please refer to the commmon causes for the performance issues by referring to this documentation - https://ds-modules.github.io/curriculum-guide/workflow/performance_issue.html. If the above mentioned reasons are not the cause for the performance issues in the hub then or you lack clarity about the issue then please do raise a request by filling the details below: + Please refer to the common causes for the performance issues by referring to this documentation - https://ds-modules.github.io/curriculum-guide/workflow/performance_issue.html. If the above mentioned reasons are not the cause for the performance issues in the hub then or you lack clarity about the issue then please do raise a request by filling the details below: - type: dropdown attributes: label: Your Role @@ -91,7 +91,7 @@ body: - type: input attributes: label: Issue timeline - description: Mention the date/time when this issue occured + description: Mention the date/time when this issue occurred validations: required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/resourcescheduler.yml b/.github/ISSUE_TEMPLATE/resourcescheduler.yml index 40ea57454..b7eb2ae84 100644 --- a/.github/ISSUE_TEMPLATE/resourcescheduler.yml +++ b/.github/ISSUE_TEMPLATE/resourcescheduler.yml @@ -24,7 +24,7 @@ body: label: Detailed Requirements description: Rationale for requesting increased compute value: | - - + - validations: required: true - type: dropdown @@ -40,7 +40,7 @@ body: - type: textarea attributes: label: Request Deadline - description: Sepcify the time duration for which you are expecting this increase in compute + description: Specify the time duration for which you are expecting this increase in compute value: | validations: diff --git a/deployments/a11y/secrets/gke-key.json b/deployments/a11y/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/a11y/secrets/gke-key.json +++ b/deployments/a11y/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/astro/secrets/gke-key.json b/deployments/astro/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/astro/secrets/gke-key.json +++ b/deployments/astro/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/biology/secrets/gke-key.json b/deployments/biology/secrets/gke-key.json index 28144a347..e14815100 100644 --- a/deployments/biology/secrets/gke-key.json +++ b/deployments/biology/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/cee/secrets/gke-key.json b/deployments/cee/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/cee/secrets/gke-key.json +++ b/deployments/cee/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/data100/secrets/gke-key.json b/deployments/data100/secrets/gke-key.json index 511dea2ad..9b71125e4 100644 --- a/deployments/data100/secrets/gke-key.json +++ b/deployments/data100/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/data101/secrets/gke-key.json b/deployments/data101/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/data101/secrets/gke-key.json +++ b/deployments/data101/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/data102/secrets/gke-key.json b/deployments/data102/secrets/gke-key.json index 2a0e2eb3d..f6a43eb97 100644 --- a/deployments/data102/secrets/gke-key.json +++ b/deployments/data102/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/datahub/secrets/gke-key.json b/deployments/datahub/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/datahub/secrets/gke-key.json +++ b/deployments/datahub/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/dev/secrets/gke-key.json b/deployments/dev/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/dev/secrets/gke-key.json +++ b/deployments/dev/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/dlab/secrets/gke-key.json b/deployments/dlab/secrets/gke-key.json index 28ca6cc5b..fdced4c6e 100644 --- a/deployments/dlab/secrets/gke-key.json +++ b/deployments/dlab/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/edx/config/common.yaml b/deployments/edx/config/common.yaml index 15d59f6a9..02f49411b 100644 --- a/deployments/edx/config/common.yaml +++ b/deployments/edx/config/common.yaml @@ -77,5 +77,3 @@ jupyterhub: extraEnv: # Unset NotebookApp from hub/values. Necessary for recent lab versions. JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp" - - diff --git a/deployments/edx/hubploy.yaml b/deployments/edx/hubploy.yaml index 4ecb78bc9..3d3de2228 100644 --- a/deployments/edx/hubploy.yaml +++ b/deployments/edx/hubploy.yaml @@ -9,5 +9,3 @@ cluster: service_key: gcloud-service-key.json cluster: edx zone: us-central1 - - diff --git a/deployments/edx/secrets/gcloud-service-key.json b/deployments/edx/secrets/gcloud-service-key.json index 3b1f43fe6..064d97266 100644 --- a/deployments/edx/secrets/gcloud-service-key.json +++ b/deployments/edx/secrets/gcloud-service-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/eecs/config/common.yaml b/deployments/eecs/config/common.yaml index 244819b3e..cab1160d2 100644 --- a/deployments/eecs/config/common.yaml +++ b/deployments/eecs/config/common.yaml @@ -103,4 +103,4 @@ jupyterhub: course::1537177: # ELENG 120, https://github.com/berkeley-dsep-infra/datahub/issues/6015 mem_limit: 4096M - mem_guarantee: 4096M \ No newline at end of file + mem_guarantee: 4096M diff --git a/deployments/eecs/secrets/gke-key.json b/deployments/eecs/secrets/gke-key.json index 3a6778da3..13043c0d1 100644 --- a/deployments/eecs/secrets/gke-key.json +++ b/deployments/eecs/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/gradebook/secrets/gke-key.json b/deployments/gradebook/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/gradebook/secrets/gke-key.json +++ b/deployments/gradebook/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/highschool/secrets/gke-key.json b/deployments/highschool/secrets/gke-key.json index 6ee727660..d330fc7e9 100644 --- a/deployments/highschool/secrets/gke-key.json +++ b/deployments/highschool/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/ischool/secrets/gke-key.json b/deployments/ischool/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/ischool/secrets/gke-key.json +++ b/deployments/ischool/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/julia/secrets/gke-key.json b/deployments/julia/secrets/gke-key.json index 9bb71f8a8..3a2a6b4dc 100644 --- a/deployments/julia/secrets/gke-key.json +++ b/deployments/julia/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/logodev/secrets/gke-key.json b/deployments/logodev/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/logodev/secrets/gke-key.json +++ b/deployments/logodev/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/nature/secrets/gke-key.json b/deployments/nature/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/nature/secrets/gke-key.json +++ b/deployments/nature/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/prob140/secrets/gke-key.json b/deployments/prob140/secrets/gke-key.json index d7a03b9c7..7364be93c 100644 --- a/deployments/prob140/secrets/gke-key.json +++ b/deployments/prob140/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/publichealth/secrets/gke-key.json b/deployments/publichealth/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/publichealth/secrets/gke-key.json +++ b/deployments/publichealth/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/r/config/common.yaml b/deployments/r/config/common.yaml index ebe79685b..cef31ce65 100644 --- a/deployments/r/config/common.yaml +++ b/deployments/r/config/common.yaml @@ -99,4 +99,3 @@ jupyterhub: course::1537932: # Data C131A, https://github.com/berkeley-dsep-infra/datahub/issues/6121 mem_limit: 2048M mem_guarantee: 2048M - diff --git a/deployments/r/secrets/gke-key.json b/deployments/r/secrets/gke-key.json index f59e3475a..7019fb575 100644 --- a/deployments/r/secrets/gke-key.json +++ b/deployments/r/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/shiny/secrets/gke-key.json b/deployments/shiny/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/shiny/secrets/gke-key.json +++ b/deployments/shiny/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/stat159/secrets/gke-key.json b/deployments/stat159/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/stat159/secrets/gke-key.json +++ b/deployments/stat159/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/stat20/secrets/gke-key.json b/deployments/stat20/secrets/gke-key.json index b93212fa3..7776709cc 100644 --- a/deployments/stat20/secrets/gke-key.json +++ b/deployments/stat20/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/ugr01/secrets/gke-key.json b/deployments/ugr01/secrets/gke-key.json index 511dea2ad..9b71125e4 100644 --- a/deployments/ugr01/secrets/gke-key.json +++ b/deployments/ugr01/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/deployments/workshop/secrets/gke-key.json b/deployments/workshop/secrets/gke-key.json index c63a0697a..a29bb7efd 100644 --- a/deployments/workshop/secrets/gke-key.json +++ b/deployments/workshop/secrets/gke-key.json @@ -27,4 +27,4 @@ "unencrypted_suffix": "_unencrypted", "version": "3.7.0" } -} \ No newline at end of file +} diff --git a/docs/admins/cluster-config.qmd b/docs/admins/cluster-config.qmd index c2d665fc1..f042ee4ea 100644 --- a/docs/admins/cluster-config.qmd +++ b/docs/admins/cluster-config.qmd @@ -18,7 +18,7 @@ In our experience, Google Kubernetes Engine (GKE) has been the most stable, performant, and reliable managed kubernetes service. We prefer running on this when possible. -A `gcloud container clusters create` command can succintly express the +A `gcloud container clusters create` command can succinctly express the configuration of our kubernetes cluster. The following command represents the currently favored configuration. diff --git a/docs/admins/howto/calendar-scaler.qmd b/docs/admins/howto/calendar-scaler.qmd index 47304c872..35ed93eb6 100644 --- a/docs/admins/howto/calendar-scaler.qmd +++ b/docs/admins/howto/calendar-scaler.qmd @@ -101,7 +101,7 @@ Here is an example of how you can test any changes to `scaler/calendar.py` locally in the python interpreter: ``` python -# these tests will use somes dates culled from the calendar with varying numbers of events. +# these tests will use some dates culled from the calendar with varying numbers of events. import scaler.calendar import datetime import zoneinfo diff --git a/docs/admins/howto/course-config.qmd b/docs/admins/howto/course-config.qmd index 939e641d1..bced44e6c 100644 --- a/docs/admins/howto/course-config.qmd +++ b/docs/admins/howto/course-config.qmd @@ -4,7 +4,7 @@ title: Course Configuration ## Allocating Resources -It is possible to alter administrative priviliges or resources +It is possible to alter administrative privileges or resources allocations (such as memory or extra volumes) of user servers from within the deployment configuration. This is mostly useful for when resources need to be increased based on users\' class enrollments. The @@ -30,7 +30,7 @@ Groups are named with the format \"course::234567::enrollment_type::student\". Our custom kubespawner, which we define in `hub/values.yaml`, reads users\' group memberships prior to spawning. It then overrides various KubeSpawner -paramters based on configuration we define, using the canvas ID as the +parameters based on configuration we define, using the canvas ID as the key. (see below) Note that if a user is assigned to a new Canvas group (e.g. by the @@ -96,7 +96,7 @@ with bCourse ID `1234567`. It assigns scopes for accessing and administering the servers for users in group `course::1234567`. Members of that group include all students and course staff. It also assigns scopes for viewing lists of -users at /hub/admin. It assignes these scopes to members of the +users at /hub/admin. It assigns these scopes to members of the affiliated course staff groups. This stanza is more verbose than inserting lists of users under diff --git a/docs/admins/howto/managing-multiple-user-image-repos.qmd b/docs/admins/howto/managing-multiple-user-image-repos.qmd index 57a1afda8..b1a9677bd 100644 --- a/docs/admins/howto/managing-multiple-user-image-repos.qmd +++ b/docs/admins/howto/managing-multiple-user-image-repos.qmd @@ -186,7 +186,7 @@ options: Commit message to use for the changes. ``` -`stage` combines both `git add ...` and `git commit -m`, adding and commiting +`stage` combines both `git add ...` and `git commit -m`, adding and committing one or more files to the staging area before you push to a remote. The commit message must be a text string enclosed in quotes. @@ -269,4 +269,4 @@ After staging, push everything to a remote: ``` manage-repos -c repos.txt -d ~/src/images push -b test-branch -``` \ No newline at end of file +``` diff --git a/docs/admins/howto/new-hub.qmd b/docs/admins/howto/new-hub.qmd index 0d3c62de8..2b598ef9c 100644 --- a/docs/admins/howto/new-hub.qmd +++ b/docs/admins/howto/new-hub.qmd @@ -75,7 +75,7 @@ you can skip either or both of the following steps and pick back up at the `cookiecutter`. When creating a new hub, we also make sure to label the filestore and -GKE/node pool resouces with both `hub` and +GKE/node pool resources with both `hub` and `-deployment`. 99.999% of the time, the values for all three of these labels will be ``. @@ -331,7 +331,7 @@ will be used to allocate RAM for the node placeholder. | egrep -v 'pause|notebook' ``` -1. Subract the second number from the first, and then subtract another +1. Subtract the second number from the first, and then subtract another 277872640 bytes (256Mi) for "wiggle room". 2. Add an entry for the new placeholder node config in `values.yaml`: diff --git a/docs/admins/howto/new-packages.qmd b/docs/admins/howto/new-packages.qmd index 06e83a697..eae812e52 100644 --- a/docs/admins/howto/new-packages.qmd +++ b/docs/admins/howto/new-packages.qmd @@ -9,7 +9,7 @@ request that the change by installed for all users by by [creating a new issue in github](https://github.com/berkeley-dsep-infra/datahub/issues),contacting cirriculum support staff, or creating a new pull request. Ultimately, -thouroughly testing changes locally and submitting a pull request will +thoroughly testing changes locally and submitting a pull request will result in the software being rolled out to everyone much faster. Install a python package in your notebook diff --git a/docs/admins/howto/rebuild-hub-image.qmd b/docs/admins/howto/rebuild-hub-image.qmd index 7c385a0d5..5df136ff4 100644 --- a/docs/admins/howto/rebuild-hub-image.qmd +++ b/docs/admins/howto/rebuild-hub-image.qmd @@ -19,7 +19,7 @@ which also updates `hub/values.yaml` with the new image version. 3. Run `chartpress --push`. This will build and push the hub image, and modify `hub/values.yaml` appropriately. 4. Make a commit with the `hub/values.yaml` file, so the new hub image - name and tag are comitted. + name and tag are committed. 5. Proceed to deployment as normal. Some of the following commands may be required to configure your diff --git a/docs/admins/howto/rebuild-postgres-image.qmd b/docs/admins/howto/rebuild-postgres-image.qmd index bd5ec8c65..37a9ab46e 100644 --- a/docs/admins/howto/rebuild-postgres-image.qmd +++ b/docs/admins/howto/rebuild-postgres-image.qmd @@ -2,7 +2,7 @@ title: "Customize the Per-User Postgres Docker Image" --- -We provide each student on `data100` witha postgresql server. We want the +We provide each student on `data100` with a postgresql server. We want the [python extension](https://www.postgresql.org/docs/current/plpython.html) installed. So we inherit from the [upstream postgresql docker image](https://hub.docker.com/_/postgres), and add the appropriate @@ -13,7 +13,7 @@ rebuild and push it. 1. Modify the image in `images/postgres` and make a git commit. 2. Run `chartpress --push`. This will build and push the image, *but - not put anything in YAML*. There is no place we can put thi in + not put anything in YAML*. There is no place we can put this in `values.yaml`, since this is only used for data100. 3. Notice the image name + tag from the `chartpress --push` command, and put it in the appropriate place (under `extraContainers`) in diff --git a/docs/datahub.svg b/docs/datahub.svg index 58439789f..46d84ae98 100644 --- a/docs/datahub.svg +++ b/docs/datahub.svg @@ -16,4 +16,4 @@ - \ No newline at end of file + diff --git a/docs/incidents/2017-02-24-autoscaler-incident.qmd b/docs/incidents/2017-02-24-autoscaler-incident.qmd index 94ee6383d..700b2a235 100644 --- a/docs/incidents/2017-02-24-autoscaler-incident.qmd +++ b/docs/incidents/2017-02-24-autoscaler-incident.qmd @@ -28,5 +28,3 @@ A cluster autoscaler script was accidentally run against the production cluster ### Future organizational change ### 1. Use a separate billing account for production vs development clusters. This makes it harder to accidentally run things on the wrong cluster - - diff --git a/docs/incidents/2017-03-20-too-many-volumes.qmd b/docs/incidents/2017-03-20-too-many-volumes.qmd index f7cea129c..1ca989360 100644 --- a/docs/incidents/2017-03-20-too-many-volumes.qmd +++ b/docs/incidents/2017-03-20-too-many-volumes.qmd @@ -11,7 +11,7 @@ From sometime early March 20 2017 till about 1300, some new student servers were ### March 18, 16:30 ## -RAM per student is [reduced](https://github.com/data-8/infrastructure/issues/16) from 2G to 1G, as a resource optimization measure. The size of our nodes remains the same (26G RAM), and many are cordonned off and slowly decomissioned over the coming few days. +RAM per student is [reduced](https://github.com/data-8/infrastructure/issues/16) from 2G to 1G, as a resource optimization measure. The size of our nodes remains the same (26G RAM), and many are cordonned off and slowly decommissioned over the coming few days. Life seems fine, given the circumstances. @@ -39,7 +39,7 @@ This new instance group was then set to expand for 30 new nodes, which will prov ### 13:04 ### -The simple autoscaler is stopped, on fear that it'll be confused by the unusal mixed state of the nodes and do something wonky. +The simple autoscaler is stopped, on fear that it'll be confused by the unusual mixed state of the nodes and do something wonky. ### 13:11 ### diff --git a/docs/incidents/2017-03-23-kernel-deaths-incident.qmd b/docs/incidents/2017-03-23-kernel-deaths-incident.qmd index 050b65050..28b95b68d 100644 --- a/docs/incidents/2017-03-23-kernel-deaths-incident.qmd +++ b/docs/incidents/2017-03-23-kernel-deaths-incident.qmd @@ -47,7 +47,7 @@ https://github.com/data-8/jupyterhub-k8s/pull/146 is identified as the culprit. ### 17:25 ### A very involved and laborious revert of the offending part of the patch is done in https://github.com/jupyterhub/kubespawner/pull/37. Core Jupyter Notebook dev continues to confirm this makes no sense. -https://github.com/data-8/jupyterhub-k8s/pull/152 is also merged, and deployed shortly after verifiying that everything (including starting kernels & executing code) works fine on dev. Deployed to prod and everything is fine. +https://github.com/data-8/jupyterhub-k8s/pull/152 is also merged, and deployed shortly after verifying that everything (including starting kernels & executing code) works fine on dev. Deployed to prod and everything is fine. ## Conclusion ## diff --git a/docs/incidents/2019-05-01-service-account-leak.qmd b/docs/incidents/2019-05-01-service-account-leak.qmd index d66bd7903..a8ff273e4 100644 --- a/docs/incidents/2019-05-01-service-account-leak.qmd +++ b/docs/incidents/2019-05-01-service-account-leak.qmd @@ -19,7 +19,7 @@ Deployments are paused until this was fixed. ### May 1 2019, 3:18 PM A template + documentation for creating new hubs easily is -pushed to GitHub as a pull request. This inadvertantly +pushed to GitHub as a pull request. This inadvertently contained live credentials for pushing & pulling our (already public) docker images, and for access to our kubernetes clusters. diff --git a/docs/incidents/2022-01-20-package-dependency-upgrade-incident.qmd b/docs/incidents/2022-01-20-package-dependency-upgrade-incident.qmd index 5b3e0a525..6000e1cde 100644 --- a/docs/incidents/2022-01-20-package-dependency-upgrade-incident.qmd +++ b/docs/incidents/2022-01-20-package-dependency-upgrade-incident.qmd @@ -9,7 +9,7 @@ date: 2022-01-20 Due to these changes, image rebuild happened which broke multiple hubs which used that image including Datahub, ISchool, R, Data 100 and Data 140 hubs. -One of the dependenices highlighted as part of the image build had an upgrade which resulted in R hub throwing 505 error and Data 100/140 hub throwing "Error starting Kernel". [Yuvi to fill in the right technical information] +One of the dependencies highlighted as part of the image build had an upgrade which resulted in R hub throwing 505 error and Data 100/140 hub throwing "Error starting Kernel". [Yuvi to fill in the right technical information] User Impact: diff --git a/docs/incidents/2024-core-node-incidents.qmd b/docs/incidents/2024-core-node-incidents.qmd index d2febd165..67613b140 100644 --- a/docs/incidents/2024-core-node-incidents.qmd +++ b/docs/incidents/2024-core-node-incidents.qmd @@ -95,7 +95,7 @@ chp restarts (no heap error) 5.7K 503 errors ### 16:54:15 - 17:15:31 -300 users (slowly descreasing), 3x chp “uncaught exception: write EPIPE”, intermittent 503 errors in spikes of 30, 60, 150, hub latency 2.5sec +300 users (slowly decreasing), 3x chp “uncaught exception: write EPIPE”, intermittent 503 errors in spikes of 30, 60, 150, hub latency 2.5sec ### 18:47:19 - 18:58:10 ~120 users (constant), 3x chp “uncaught exception: write EPIPE”, intermittent 503 errors in spikes of 30, 60, hub latency 3sec diff --git a/docs/policy/create_policy.qmd b/docs/policy/create_policy.qmd index 902daa202..789dda01b 100644 --- a/docs/policy/create_policy.qmd +++ b/docs/policy/create_policy.qmd @@ -2,4 +2,4 @@ - The policy subfolder inside the docs folder in the datahub repository acts as a single source of truth for all our policy-related proposals. - Every PR related to a specific policy will go through multiple reviews from the team before getting finalized and merged. Reviewers should be explicitly assigned in order to seek the required approval prior to merging the PR. -- Each policy issue area will get mapped to a specific policy document. Eg: The policy to create a new hub is currently mapped to the policy_create_hubs.rst file. \ No newline at end of file +- Each policy issue area will get mapped to a specific policy document. Eg: The policy to create a new hub is currently mapped to the policy_create_hubs.rst file. diff --git a/docs/policy/policy_deploy_mainhubs.qmd b/docs/policy/policy_deploy_mainhubs.qmd index 64f14a6f6..d3098ab45 100644 --- a/docs/policy/policy_deploy_mainhubs.qmd +++ b/docs/policy/policy_deploy_mainhubs.qmd @@ -4,4 +4,4 @@ Our goal is to provide a reliable infrastructure that instructors can completely trust while facilitating their coursework. Developing a robust protocol around deploying changes to the main datahub is important to achieve this goal. The objective of this policy document is to outline the criteria to deploy a change to an image in the main Datahub. - Regular requests during the semester like package addition/change, RAM increase, CPU allocation, and providing admin access to users should be done with a robust testing protocol (either automated or manual) in place across staging and production. -- Introduce new features in the main Datahub only after it gets successfully tested with one or many instructors across other course specific hub (eg: Data8 Hub) or use-case specific hub (Eg: Stat159 Hub). \ No newline at end of file +- Introduce new features in the main Datahub only after it gets successfully tested with one or many instructors across other course specific hub (eg: Data8 Hub) or use-case specific hub (Eg: Stat159 Hub). diff --git a/docs/policy/principles.qmd b/docs/policy/principles.qmd index e79584c91..99bd58f33 100644 --- a/docs/policy/principles.qmd +++ b/docs/policy/principles.qmd @@ -22,4 +22,4 @@ We want to be the best stewards of CDSS money. We value being thoughtful and fru ## Resourcefulness -We are not a big team, and we don't have a lot of experts related to Marketing, Design, Data Science, etc. However, we want to be resourceful with the limited resources we have and engage with the open-source ecosystem, our partners across Berkeley, and the instructors/students to work towards our mission. \ No newline at end of file +We are not a big team, and we don't have a lot of experts related to Marketing, Design, Data Science, etc. However, we want to be resourceful with the limited resources we have and engage with the open-source ecosystem, our partners across Berkeley, and the instructors/students to work towards our mission. diff --git a/docs/users/hubs/stat159.qmd b/docs/users/hubs/stat159.qmd index ea559dd3a..afc2280a6 100644 --- a/docs/users/hubs/stat159.qmd +++ b/docs/users/hubs/stat159.qmd @@ -56,7 +56,7 @@ in secrets. This configuration is to encourage use of RTC, and to prevent one student from having too much access to another student\'s home -directory. The fabricated (essentially service) accounts have initally +directory. The fabricated (essentially service) accounts have initially empty home directories and exist solely to provide workspaces for the group. There is currently no archive or restore procedure in mind for these shared accounts. diff --git a/images/node-placeholder-scaler/scaler/__main__.py b/images/node-placeholder-scaler/scaler/__main__.py index bfb5c7513..f317f6fda 100644 --- a/images/node-placeholder-scaler/scaler/__main__.py +++ b/images/node-placeholder-scaler/scaler/__main__.py @@ -1,5 +1,4 @@ from .scaler import main - if __name__ == "__main__": main() diff --git a/images/node-placeholder-scaler/scaler/calendar.py b/images/node-placeholder-scaler/scaler/calendar.py index d494620f4..cb92390d8 100755 --- a/images/node-placeholder-scaler/scaler/calendar.py +++ b/images/node-placeholder-scaler/scaler/calendar.py @@ -2,9 +2,9 @@ import datetime import logging import re -import niquests as requests import zoneinfo +import niquests as requests from ical.calendar_stream import IcsCalendarStream log = logging.getLogger(__name__) diff --git a/images/node-placeholder-scaler/scaler/scaler.py b/images/node-placeholder-scaler/scaler/scaler.py index 8265618bf..b42f3390d 100755 --- a/images/node-placeholder-scaler/scaler/scaler.py +++ b/images/node-placeholder-scaler/scaler/scaler.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -import logging import argparse -import tempfile -import subprocess -import niquests as requests +import logging import os -from copy import deepcopy +import subprocess +import tempfile import time +from copy import deepcopy -from .calendar import get_calendar, get_events, _event_repr - +import niquests as requests from ruamel.yaml import YAML +from .calendar import _event_repr, get_calendar, get_events + yaml = YAML(typ="safe") diff --git a/node-placeholder/templates/configmap.yaml b/node-placeholder/templates/configmap.yaml index a8f543f98..f77d0f6cb 100644 --- a/node-placeholder/templates/configmap.yaml +++ b/node-placeholder/templates/configmap.yaml @@ -6,4 +6,4 @@ metadata: {{- include "node-placeholder-scaler.labels" . | nindent 4 }} data: placeholder-template.yaml: {{ include "node-placeholder-scaler.placeholderTemplate" . | quote }} - config.yaml: {{ .Values | toJson | quote }} \ No newline at end of file + config.yaml: {{ .Values | toJson | quote }} diff --git a/node-placeholder/templates/priorityclass.yaml b/node-placeholder/templates/priorityclass.yaml index 460ee13bf..22af485d2 100644 --- a/node-placeholder/templates/priorityclass.yaml +++ b/node-placeholder/templates/priorityclass.yaml @@ -7,6 +7,6 @@ metadata: value: -10 globalDefault: false description: | - With a priority higher or eqaul to a cluster autoscalers + With a priority higher or equal to a cluster autoscalers priority cutoff, a pod can trigger a cluster scale up. At the same time, - placeholder pods priority should be lower than other pods to make them evictable \ No newline at end of file + placeholder pods priority should be lower than other pods to make them evictable diff --git a/node-placeholder/templates/rbac.yaml b/node-placeholder/templates/rbac.yaml index 449f0c8b6..475b6653a 100644 --- a/node-placeholder/templates/rbac.yaml +++ b/node-placeholder/templates/rbac.yaml @@ -23,4 +23,4 @@ roleRef: kind: Role name: {{ include "node-placeholder-scaler.fullname" . }} apiGroup: rbac.authorization.k8s.io -{{- end}} \ No newline at end of file +{{- end}} diff --git a/requirements.txt b/requirements.txt index 569761591..7b6d448bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -git+https://github.com/berkeley-dsep-infra/hubploy.git -pygithub -pyrsistent==0.19.3 attrs==23.1.0 +chardet chartpress +git+https://github.com/berkeley-dsep-infra/hubploy.git jupyter-repo2docker==2024.07.0 myst-parser -chardet niquests==3.7.2 +pygithub +pyrsistent==0.19.3 urllib3<2.0.0 yamllint==1.35.1 diff --git a/scripts/delete-unused-users.py b/scripts/delete-unused-users.py index f13687cc4..1b9c5095f 100755 --- a/scripts/delete-unused-users.py +++ b/scripts/delete-unused-users.py @@ -16,13 +16,13 @@ """ import argparse -from datetime import timedelta, datetime import json import logging import os -import niquests import sys +from datetime import datetime, timedelta +import niquests from dateutil.parser import parse logging.basicConfig(stream=sys.stdout, level=logging.WARNING) @@ -77,7 +77,7 @@ def retrieve_users(hub_url, headers, inactive_since): def should_delete(user, inactive_since): """ - Returns a boolean if user is to be deleted. The critera are: + Returns a boolean if user is to be deleted. The criteria are: - was the user active in the past inactive_since period? - is there a current user server running? """ diff --git a/scripts/git-pre-cloner.py b/scripts/git-pre-cloner.py index 667110958..11f378a0e 100644 --- a/scripts/git-pre-cloner.py +++ b/scripts/git-pre-cloner.py @@ -40,7 +40,7 @@ def copy_repo(username): dest_dir = os.path.join(home_dir, repo_dirname) if os.path.exists(dest_dir): if args.verbose: - print("Skipping {}".format(safe)) + print(f"Skipping {safe}") else: if args.verbose: print(safe) diff --git a/scripts/infra-packages/requirements.txt b/scripts/infra-packages/requirements.txt index 0fb0bd930..b410ce6fd 100644 --- a/scripts/infra-packages/requirements.txt +++ b/scripts/infra-packages/requirements.txt @@ -3,27 +3,27 @@ # After editing scripts/infra-packages/requirements.txt, please run # scripts/infra-packages/sync.bash. +appmode==0.8.0 +# Measure popularity of different packages in our hubs +# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda +git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling +git-credential-helpers==0.2 +ipywidgets==8.0.7 +# jupyter archival tool for easy user downloads +jupyter-archive==3.4.0 +jupyter-resource-usage==1.0.1 +# syncthing for dropbox-like functionality +jupyter-syncthing-proxy==1.0.3 +jupyter-tree-download==1.0.1 +# Matches version in images/hub/Dockerfile +jupyterhub==4.1.6 +jupyterlab==4.0.11 +nbgitpuller==1.2.1 # This file pins versions of notebook related python packages we want # across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. +# everywhere one by one. # FIXME: Freeze this to get exact versions of all dependencies notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/scripts/list-packages.py b/scripts/list-packages.py index b825804ed..55cd44d43 100755 --- a/scripts/list-packages.py +++ b/scripts/list-packages.py @@ -3,11 +3,12 @@ Lists R packages in one docker image but not the other """ -import docker import argparse import json -from urllib.request import urlopen from urllib.error import HTTPError +from urllib.request import urlopen + +import docker argparser = argparse.ArgumentParser() argparser.add_argument( @@ -52,7 +53,7 @@ def packages_list(image_name): .split("\n")[2:] ) - return set([rp.split()[0] for rp in raw_packages if len(rp.split()) == 3]) + return {rp.split()[0] for rp in raw_packages if len(rp.split()) == 3} def main(): diff --git a/scripts/post-grafana-annotation.py b/scripts/post-grafana-annotation.py index 888555b90..c2c2da90c 100644 --- a/scripts/post-grafana-annotation.py +++ b/scripts/post-grafana-annotation.py @@ -14,10 +14,11 @@ """ import argparse -import niquests as requests import os import time +import niquests as requests + def create_annotation(grafana_url, grafana_api_key, tags, text): """ diff --git a/scripts/rsync-active-users.py b/scripts/rsync-active-users.py index 797402a64..8e9c621a4 100755 --- a/scripts/rsync-active-users.py +++ b/scripts/rsync-active-users.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """ rsync home directories of active users only. @@ -10,20 +11,21 @@ obtainable from {hub_url}/hub/token by an admin user. """ -import os -import niquests as requests -from dateutil.parser import parse -from datetime import datetime, timedelta, timezone -from concurrent.futures import ThreadPoolExecutor, as_completed import argparse -import time +import os +import string import subprocess +import sys +import time # Copied from https://github.com/minrk/escapism/blob/d1d406c69b9ab0b14aa562d98a9e198adf9c047a/escapism.py # this is the library JuptyerHub uses to escape usernames into a form that works for filesystem paths import warnings -import string -import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timedelta, timezone + +import niquests as requests +from dateutil.parser import parse SAFE = set(string.ascii_letters + string.digits) ESCAPE_CHAR = "_" diff --git a/scripts/user-image-management/README.md b/scripts/user-image-management/README.md index 11bad3a2e..ad1a318d7 100644 --- a/scripts/user-image-management/README.md +++ b/scripts/user-image-management/README.md @@ -1,3 +1,3 @@ -# Tooling to help mange user image repos +# Tooling to help manage user image repos -See the official [documentation](https://docs.datahub.berkeley.edu/admins/howto/managing-multiple-user-image-repos.html) \ No newline at end of file +See the official [documentation](https://docs.datahub.berkeley.edu/admins/howto/managing-multiple-user-image-repos.html) diff --git a/scripts/user-image-management/manage-image-repos.py b/scripts/user-image-management/manage-image-repos.py index 1fad8a4ba..b894af392 100755 --- a/scripts/user-image-management/manage-image-repos.py +++ b/scripts/user-image-management/manage-image-repos.py @@ -7,9 +7,9 @@ """ import argparse +import os import subprocess import sys -import os def clone(args): diff --git a/vendor/google/ugr/README.md b/vendor/google/ugr/README.md index 7d1667975..bc6adc817 100644 --- a/vendor/google/ugr/README.md +++ b/vendor/google/ugr/README.md @@ -14,7 +14,7 @@ gcloud deployment-manager deployments create ... || \ gcloud deployment-manager deployments update ... ``` -That is, the script will run the create version; if this fails (due to the deployement already existing), then it will update the deployment, only taking actions to effect changes in the "deltas" between the live deployment and the new local configuration. +That is, the script will run the create version; if this fails (due to the deployment already existing), then it will update the deployment, only taking actions to effect changes in the "deltas" between the live deployment and the new local configuration. Deployments may be destroyed wholesale with "one click" in the web console, or with gcloud as described in the shell script comments: @@ -52,7 +52,7 @@ When choosing python for the template language, the following function is expect def GenerateConfig(context): ``` -At deployment, the template has access to the properties set in the associated yaml, and this function's one job is to return a dictionary that descries the deployment: +At deployment, the template has access to the properties set in the associated yaml, and this function's one job is to return a dictionary that describes the deployment: ``` return {'resources': resources} diff --git a/vendor/grafana/dashboards/general/Anomalous activities-1714066292129.json b/vendor/grafana/dashboards/general/Anomalous activities-1714066292129.json index 533dbba49..bb610098f 100644 --- a/vendor/grafana/dashboards/general/Anomalous activities-1714066292129.json +++ b/vendor/grafana/dashboards/general/Anomalous activities-1714066292129.json @@ -270,4 +270,4 @@ "uid": "MR7_7MwZk", "version": 2, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/Cloud Storage Monitoring-1714066317471.json b/vendor/grafana/dashboards/general/Cloud Storage Monitoring-1714066317471.json index 25c536612..98ad8c441 100644 --- a/vendor/grafana/dashboards/general/Cloud Storage Monitoring-1714066317471.json +++ b/vendor/grafana/dashboards/general/Cloud Storage Monitoring-1714066317471.json @@ -918,4 +918,4 @@ "uid": "0Xx3Zao4z", "version": 2, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/Cluster Health-1714066323007.json b/vendor/grafana/dashboards/general/Cluster Health-1714066323007.json index fd0925bd4..081f9456f 100644 --- a/vendor/grafana/dashboards/general/Cluster Health-1714066323007.json +++ b/vendor/grafana/dashboards/general/Cluster Health-1714066323007.json @@ -1307,4 +1307,4 @@ "uid": "naRCw05Wz", "version": 12, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/Cluster Information2-1714066330142.json b/vendor/grafana/dashboards/general/Cluster Information2-1714066330142.json index 4c9d3dbc4..488521916 100644 --- a/vendor/grafana/dashboards/general/Cluster Information2-1714066330142.json +++ b/vendor/grafana/dashboards/general/Cluster Information2-1714066330142.json @@ -1184,4 +1184,4 @@ "uid": "R3O4mbg7z", "version": 2, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/Core pods resource usage-1714066335185.json b/vendor/grafana/dashboards/general/Core pods resource usage-1714066335185.json index 7ab715055..117763986 100644 --- a/vendor/grafana/dashboards/general/Core pods resource usage-1714066335185.json +++ b/vendor/grafana/dashboards/general/Core pods resource usage-1714066335185.json @@ -298,4 +298,4 @@ "uid": "tdPns4WMk", "version": 4, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/HTTP Requests (nginx) Copy-1714066338895.json b/vendor/grafana/dashboards/general/HTTP Requests (nginx) Copy-1714066338895.json index 0675a4204..f570c8004 100644 --- a/vendor/grafana/dashboards/general/HTTP Requests (nginx) Copy-1714066338895.json +++ b/vendor/grafana/dashboards/general/HTTP Requests (nginx) Copy-1714066338895.json @@ -310,4 +310,4 @@ "uid": "IxT315H7z", "version": 2, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/Hub Metrics-1714066344166.json b/vendor/grafana/dashboards/general/Hub Metrics-1714066344166.json index 50ea27212..38635c43f 100644 --- a/vendor/grafana/dashboards/general/Hub Metrics-1714066344166.json +++ b/vendor/grafana/dashboards/general/Hub Metrics-1714066344166.json @@ -1867,4 +1867,4 @@ "uid": "GqFAtZRZz", "version": 6, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/JupyterHub Dashboard2-1714066348840.json b/vendor/grafana/dashboards/general/JupyterHub Dashboard2-1714066348840.json index b7dc599c6..92d049759 100644 --- a/vendor/grafana/dashboards/general/JupyterHub Dashboard2-1714066348840.json +++ b/vendor/grafana/dashboards/general/JupyterHub Dashboard2-1714066348840.json @@ -1390,4 +1390,4 @@ "uid": "hub-dashboard2", "version": 3, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/NGINX Ingress controller-1714066353536.json b/vendor/grafana/dashboards/general/NGINX Ingress controller-1714066353536.json index 754373298..7b6c67446 100644 --- a/vendor/grafana/dashboards/general/NGINX Ingress controller-1714066353536.json +++ b/vendor/grafana/dashboards/general/NGINX Ingress controller-1714066353536.json @@ -1582,4 +1582,4 @@ "uid": "nginx", "version": 14, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/Python Popularity Contest-1714066357213.json b/vendor/grafana/dashboards/general/Python Popularity Contest-1714066357213.json index 5c9246851..98fa2b358 100644 --- a/vendor/grafana/dashboards/general/Python Popularity Contest-1714066357213.json +++ b/vendor/grafana/dashboards/general/Python Popularity Contest-1714066357213.json @@ -166,4 +166,4 @@ "uid": "1EJ3WNi7z", "version": 5, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/Ryan's Draft Panels-1714066360784.json b/vendor/grafana/dashboards/general/Ryan's Draft Panels-1714066360784.json index bd616f058..aaa287376 100644 --- a/vendor/grafana/dashboards/general/Ryan's Draft Panels-1714066360784.json +++ b/vendor/grafana/dashboards/general/Ryan's Draft Panels-1714066360784.json @@ -620,4 +620,4 @@ "uid": "BLAq4fv4z", "version": 18, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/Storage I_O Statistics-1714066365230.json b/vendor/grafana/dashboards/general/Storage I_O Statistics-1714066365230.json index d5c9a0323..1769ba121 100644 --- a/vendor/grafana/dashboards/general/Storage I_O Statistics-1714066365230.json +++ b/vendor/grafana/dashboards/general/Storage I_O Statistics-1714066365230.json @@ -954,4 +954,4 @@ "uid": "RzrUuuQZk", "version": 3, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/general/nginx based request metrics-1714066371135.json b/vendor/grafana/dashboards/general/nginx based request metrics-1714066371135.json index 7e01701f1..65bd873ea 100644 --- a/vendor/grafana/dashboards/general/nginx based request metrics-1714066371135.json +++ b/vendor/grafana/dashboards/general/nginx based request metrics-1714066371135.json @@ -138,4 +138,4 @@ "uid": "_NMPU5DVz", "version": 2, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/jupyterhub/Cluster Information-1714066576091.json b/vendor/grafana/dashboards/jupyterhub/Cluster Information-1714066576091.json index aa981a597..bf2e621f0 100644 --- a/vendor/grafana/dashboards/jupyterhub/Cluster Information-1714066576091.json +++ b/vendor/grafana/dashboards/jupyterhub/Cluster Information-1714066576091.json @@ -1074,4 +1074,4 @@ "uid": "ec53d43b-704e-4e30-9a01-bc015df24c6a", "version": 4, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/jupyterhub/Global Usage Dashboard-1714066534688.json b/vendor/grafana/dashboards/jupyterhub/Global Usage Dashboard-1714066534688.json index fa8583a2b..b0694d3e8 100644 --- a/vendor/grafana/dashboards/jupyterhub/Global Usage Dashboard-1714066534688.json +++ b/vendor/grafana/dashboards/jupyterhub/Global Usage Dashboard-1714066534688.json @@ -312,4 +312,4 @@ "uid": "global-usage-dashboard", "version": 5, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/jupyterhub/JupyterHub Dashboard-1714066548245.json b/vendor/grafana/dashboards/jupyterhub/JupyterHub Dashboard-1714066548245.json index 08c841232..e015aea48 100644 --- a/vendor/grafana/dashboards/jupyterhub/JupyterHub Dashboard-1714066548245.json +++ b/vendor/grafana/dashboards/jupyterhub/JupyterHub Dashboard-1714066548245.json @@ -2292,4 +2292,4 @@ "uid": "hub-dashboard", "version": 24, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/jupyterhub/NFS and Support Information-1714066550552.json b/vendor/grafana/dashboards/jupyterhub/NFS and Support Information-1714066550552.json index 3779e9aa4..213bbee78 100644 --- a/vendor/grafana/dashboards/jupyterhub/NFS and Support Information-1714066550552.json +++ b/vendor/grafana/dashboards/jupyterhub/NFS and Support Information-1714066550552.json @@ -1112,4 +1112,4 @@ "uid": "ef64c7e4-8e2e-4f8d-a040-6f45bd25d4f5", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/jupyterhub/Usage Report-1714066552403.json b/vendor/grafana/dashboards/jupyterhub/Usage Report-1714066552403.json index 8eade6c7b..81cf45902 100644 --- a/vendor/grafana/dashboards/jupyterhub/Usage Report-1714066552403.json +++ b/vendor/grafana/dashboards/jupyterhub/Usage Report-1714066552403.json @@ -348,4 +348,4 @@ "uid": "usage-report", "version": 2, "weekStart": "" -} \ No newline at end of file +} diff --git a/vendor/grafana/dashboards/jupyterhub/User Diagnostics Dashboard-1714066554584.json b/vendor/grafana/dashboards/jupyterhub/User Diagnostics Dashboard-1714066554584.json index 8a0835f3d..dde3f5ec8 100644 --- a/vendor/grafana/dashboards/jupyterhub/User Diagnostics Dashboard-1714066554584.json +++ b/vendor/grafana/dashboards/jupyterhub/User Diagnostics Dashboard-1714066554584.json @@ -579,4 +579,4 @@ "uid": "user-pod-diagnostics-dashboard", "version": 2, "weekStart": "" -} \ No newline at end of file +} From 63fede667a1e2680da960cc0eda09a24019d8e00 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Fri, 18 Oct 2024 11:25:00 -0700 Subject: [PATCH 12/15] Increase RAM for MCB201B --- deployments/biology/config/common.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/deployments/biology/config/common.yaml b/deployments/biology/config/common.yaml index c1d6943fe..8cebf45fe 100644 --- a/deployments/biology/config/common.yaml +++ b/deployments/biology/config/common.yaml @@ -120,9 +120,6 @@ jupyterhub: mountPath: /etc/github/github-app-private-key.pem # data in secret file - - - nodeSelector: hub.jupyter.org/pool-name: biology-pool storage: @@ -143,3 +140,13 @@ jupyterhub: # users on the nodes. So we reduce the max amount of CPU available to them, from 7 # (defined in hub/values.yaml) to 4. limit: 4 + + group_profiles: + # DataHub Infrastructure staff + # https://bcourses.berkeley.edu/courses/1524699/groups#tab-80607 + course::1524699::group::all-admins: + admin: true + + course::1537301: # MCELLBI 201B, https://github.com/berkeley-dsep-infra/datahub/issues/6385 + mem_limit: 5120M + mem_guarantee: 5120M From d4179c770acb0a64c1059dc9af13b8d7bccec390 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Fri, 18 Oct 2024 11:50:33 -0700 Subject: [PATCH 13/15] Fixing bio stanza --- deployments/biology/config/common.yaml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/deployments/biology/config/common.yaml b/deployments/biology/config/common.yaml index 8cebf45fe..dc0160176 100644 --- a/deployments/biology/config/common.yaml +++ b/deployments/biology/config/common.yaml @@ -61,6 +61,10 @@ jupyterhub: course::1524699::group::all-admins: mem_limit: 4096M mem_guarantee: 4096M + course::1537301: # MCELLBI 201B, https://github.com/berkeley-dsep-infra/datahub/issues/6385 + mem_limit: 5120M + mem_guarantee: 5120M + # BioE C149, Fall 2024, https://github.com/berkeley-dsep-infra/datahub/issues/6205 course::1537116::enrollment_type::teacher: @@ -140,13 +144,3 @@ jupyterhub: # users on the nodes. So we reduce the max amount of CPU available to them, from 7 # (defined in hub/values.yaml) to 4. limit: 4 - - group_profiles: - # DataHub Infrastructure staff - # https://bcourses.berkeley.edu/courses/1524699/groups#tab-80607 - course::1524699::group::all-admins: - admin: true - - course::1537301: # MCELLBI 201B, https://github.com/berkeley-dsep-infra/datahub/issues/6385 - mem_limit: 5120M - mem_guarantee: 5120M From 4a983d9a426cd35452c64d13a3f5fa3f90028d25 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 18 Oct 2024 13:16:31 -0700 Subject: [PATCH 14/15] fixing up dev and deploy reqs --- dev-requirements.txt | 10 +++++----- requirements.txt | 11 +++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index f6ab4ebff..5736f9761 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,12 +1,12 @@ attrs==23.1.0 -chardet -chartpress +chardet==5.2.0 +chartpress==2.2.0 git+https://github.com/berkeley-dsep-infra/hubploy.git jupyter-repo2docker==2024.07.0 -myst-parser +myst-parser==4.0.0 niquests==3.7.2 pre-commit==4.0.1 -pygithub +pygithub==2.4.0 pyrsistent==0.19.3 -urllib3<2.0.0 +urllib3==2.2.3 yamllint==1.35.1 diff --git a/requirements.txt b/requirements.txt index 7b6d448bd..c2a419418 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,10 @@ attrs==23.1.0 -chardet -chartpress +chardet==5.2.0 +chartpress==2.2.0 git+https://github.com/berkeley-dsep-infra/hubploy.git -jupyter-repo2docker==2024.07.0 -myst-parser +myst-parser==4.0.0 niquests==3.7.2 -pygithub +pygithub==2.4.0 pyrsistent==0.19.3 -urllib3<2.0.0 +urllib3==2.2.3 yamllint==1.35.1 From 8b7c6f9b5ebacd87beb14904a2b400918ffb222c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 18 Oct 2024 13:16:58 -0700 Subject: [PATCH 15/15] infra-reqs is byebye --- scripts/infra-packages/requirements.txt | 29 ------------------------- scripts/infra-packages/sync.bash | 15 ------------- 2 files changed, 44 deletions(-) delete mode 100644 scripts/infra-packages/requirements.txt delete mode 100755 scripts/infra-packages/sync.bash diff --git a/scripts/infra-packages/requirements.txt b/scripts/infra-packages/requirements.txt deleted file mode 100644 index b410ce6fd..000000000 --- a/scripts/infra-packages/requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -appmode==0.8.0 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -git-credential-helpers==0.2 -ipywidgets==8.0.7 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 -jupyter-resource-usage==1.0.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -jupyter-tree-download==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everywhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 diff --git a/scripts/infra-packages/sync.bash b/scripts/infra-packages/sync.bash deleted file mode 100755 index 26f1b812a..000000000 --- a/scripts/infra-packages/sync.bash +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -ROOT_DIR="${SCRIPT_DIR}/../.." - -# install file in deployment directories named "image" -find "${ROOT_DIR}/deployments" -type d -name 'image' \ - -exec cp "${SCRIPT_DIR}/requirements.txt" {}/infra-requirements.txt \; - -# install file in subdirectories of deployment directories named "images" -for d in $(find "${ROOT_DIR}/deployments" -type d -name images); do - find $d -not -name images -maxdepth 1 -type d \ - -exec cp "${SCRIPT_DIR}/requirements.txt" {}/infra-requirements.txt \; -done