Skip to content

Commit

Permalink
Move the config generation out of the deployer and pilot it on staging
Browse files Browse the repository at this point in the history
  • Loading branch information
@GeorgianaElena
GeorgianaElena committed Mar 14, 2023
1 parent 3bc2ab2 commit 80cca07
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 134 deletions.
74 changes: 74 additions & 0 deletions config/clusters/2i2c/staging.values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,17 @@ staticWebsite:
enabled: false

jupyterhub:
proxy:
https:
hosts:
- staging.2i2c.cloud
ingress:
hosts:
- staging.2i2c.cloud
tls:
- secretName: https-auto-tls

This comment has been minimized.

Sign in to view

Copy link
@consideRatio

consideRatio Mar 14, 2023

I think its good if this name relates to the ingress resource, which is a jupyterhub ingress, so jupyterhub-auto-tls makes more sense than https-auto-tls I figure.
hosts:
- staging.2i2c.cloud
custom:
2i2c:
add_staff_user_ids_to_admin_users: true
Expand Down Expand Up @@ -53,3 +64,66 @@ jupyterhub:
# Only show the option to login with Google
shown_idps:
- http://google.com/accounts/o8/id
initContainers:

This comment has been minimized.

Sign in to view

Copy link
@consideRatio

consideRatio Mar 14, 2023 •
edited
Loading

Would this need to be put into all clusters' hubs' config, because we have a few variables?

I suspect there is a better approach for us to take then, such as creating a configmap with such variable values, then mounting an environment variable from the configmap etc. That way, we could put this in basehub, and let a configmap template render to include misc variables we need to reference from the container.

Looking at the initContainers and extraContainers, I see only a branch name of relevance that differs. So I figure the following:

  1. basehub adds a ConfigMap template that renders based on some variable we configure in values.yaml for basehub
  2. basehub defines hub.initContainers and hub.extraContainers, but also lets the extra container defined that references a hub specific variable do that via a ConfigMap mounted environment variable.

This comment has been minimized.

Sign in to view

Copy link
@consideRatio

consideRatio Mar 22, 2023

I wrote 2i2c-org#2349 (review) to make this comment more visible, and not lost if this commit is replaced during a rebase.
- name: templates-clone
image: alpine/git
args:
- clone
- --
- https://github.com/2i2c-org/default-hub-homepage
- /srv/repo
securityContext:
runAsUser: 1000
runAsGroup: 1000
allowPrivilegeEscalation: False
readOnlyRootFilesystem: True
volumeMounts:
- name: custom-templates
mountPath: /srv/repo
- name: templates-ownership-fix
image: alpine/git
command:
- /bin/sh
args:
- -c
- ls -lhd /srv/repo && chown 1000:1000 /srv/repo && ls -lhd /srv/repo
securityContext:
runAsUser: 0
volumeMounts:
- name: custom-templates
mountPath: /srv/repo
extraContainers:
- name: templates-sync
image: alpine/git
workingDir: /srv/repo
command:
- /bin/sh
args:
- -c
- |
ls -lhd /srv/repo;
while true; do git fetch origin;
if [[ $(git ls-remote --heads origin 2i2c-staging | wc -c) -ne 0 ]]; then
git reset --hard origin/2i2c-staging;
else
git reset --hard origin/master;
fi
sleep 5m; done
securityContext:
runAsUser: 1000
runAsGroup: 1000
allowPrivilegeEscalation: False
readOnlyRootFilesystem: True
volumeMounts:
- name: custom-templates
mountPath: /srv/repo
extraVolumes:
- name: custom-templates
emptyDir: {}
extraVolumeMounts:
- mountPath: /usr/local/share/jupyterhub/custom_templates
name: custom-templates
subPath: templates
- mountPath: /usr/local/share/jupyterhub/static/extra-assets
name: custom-templates
subPath: extra-assets
135 changes: 1 addition & 134 deletions deployer/hub.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
import json
import subprocess
import tempfile
from pathlib import Path
from textwrap import dedent

from ruamel.yaml import YAML

Expand All @@ -23,125 +20,6 @@ def __init__(self, cluster, spec):
self.cluster = cluster
self.spec = spec

def get_generated_config(self):
"""
Generate config automatically for each hub
WARNING: MIGHT CONTAINS SECRET VALUES!
"""
generated_config = {
"jupyterhub": {
"proxy": {"https": {"hosts": [self.spec["domain"]]}},
"ingress": {
"hosts": [self.spec["domain"]],
"tls": [
{
"secretName": "https-auto-tls",
"hosts": [self.spec["domain"]],
}
],
},
"hub": {
"config": {},
"initContainers": [
{
"name": "templates-clone",
"image": "alpine/git",
"args": [
"clone",
"--",
"https://github.com/2i2c-org/default-hub-homepage",
"/srv/repo",
],
"securityContext": {
"runAsUser": 1000,
"runAsGroup": 1000,
"allowPrivilegeEscalation": False,
"readOnlyRootFilesystem": True,
},
"volumeMounts": [
{
"name": "custom-templates",
"mountPath": "/srv/repo",
}
],
},
{
"name": "templates-ownership-fix",
"image": "alpine/git",
"command": ["/bin/sh"],
"args": [
"-c",
"ls -lhd /srv/repo && chown 1000:1000 /srv/repo && ls -lhd /srv/repo",
],
"securityContext": {"runAsUser": 0},
"volumeMounts": [
{
"name": "custom-templates",
"mountPath": "/srv/repo",
}
],
},
],
"extraContainers": [
{
"name": "templates-sync",
"image": "alpine/git",
"workingDir": "/srv/repo",
"command": ["/bin/sh"],
"args": [
"-c",
dedent(
f"""\
ls -lhd /srv/repo;
while true; do git fetch origin;
if [[ $(git ls-remote --heads origin {self.cluster.spec["name"]}-{self.spec["name"]} | wc -c) -ne 0 ]]; then
git reset --hard origin/{self.cluster.spec["name"]}-{self.spec["name"]};
else
git reset --hard origin/master;
fi
sleep 5m; done
"""
),
],
"securityContext": {
"runAsUser": 1000,
"runAsGroup": 1000,
"allowPrivilegeEscalation": False,
"readOnlyRootFilesystem": True,
},
"volumeMounts": [
{
"name": "custom-templates",
"mountPath": "/srv/repo",
}
],
}
],
"extraVolumes": [{"name": "custom-templates", "emptyDir": {}}],
"extraVolumeMounts": [
{
"mountPath": "/usr/local/share/jupyterhub/custom_templates",
"name": "custom-templates",
"subPath": "templates",
},
{
"mountPath": "/usr/local/share/jupyterhub/static/extra-assets",
"name": "custom-templates",
"subPath": "extra-assets",
},
],
},
},
}

# Due to nesting of charts on top of the basehub, our generated basehub
# config may need to be nested as well.
if self.spec["helm_chart"] == "daskhub":
generated_config = {"basehub": generated_config}

return generated_config

def deploy(self, dask_gateway_version):
"""
Deploy this hub
Expand All @@ -168,8 +46,6 @@ def deploy(self, dask_gateway_version):

self.spec["domain"] = domain_override_config["domain"]

generated_values = self.get_generated_config()

if self.spec["helm_chart"] == "daskhub":
# Install CRDs for daskhub before deployment
manifest_urls = [
Expand All @@ -180,15 +56,10 @@ def deploy(self, dask_gateway_version):
for manifest_url in manifest_urls:
subprocess.check_call(["kubectl", "apply", "-f", manifest_url])

with tempfile.NamedTemporaryFile(
mode="w"
) as generated_values_file, get_decrypted_files(
with get_decrypted_files(
self.cluster.config_path.joinpath(p)
for p in self.spec["helm_chart_values_files"]
) as values_files:
json.dump(generated_values, generated_values_file)
generated_values_file.flush()

cmd = [
"helm",
"upgrade",
Expand All @@ -198,10 +69,6 @@ def deploy(self, dask_gateway_version):
f"--namespace={self.spec['name']}",
self.spec["name"],
helm_charts_dir.joinpath(self.spec["helm_chart"]),
# Ordering matters here - config explicitly mentioned in cli should take
# priority over our generated values. Based on how helm does overrides, this means
# we should put the config from cluster.yaml last.
f"--values={generated_values_file.name}",
]

# Add on the values files
Expand Down

0 comments on commit 80cca07

Please sign in to comment.