From ddbfe6d6bad90095781fe96f0f10434684b420a5 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 07:54:11 +0200 Subject: [PATCH 001/128] Add Ansible inventory for GESIS stage cluster --- ansible/inventories/gesis-stage | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 ansible/inventories/gesis-stage diff --git a/ansible/inventories/gesis-stage b/ansible/inventories/gesis-stage new file mode 100644 index 000000000..a09315bf8 --- /dev/null +++ b/ansible/inventories/gesis-stage @@ -0,0 +1,41 @@ +[all] +#svko-ilcm04 ansible_host=194.95.75.14 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_14 }}' +svko-css-backup-node ansible_host=194.95.75.20 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_20 }}' +svko-k8s-test01 ansible_host=194.95.75.21 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_21 }}' +svko-k8s-test02 ansible_host=194.95.75.22 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_22 }}' +svko-k8s-test03 ansible_host=194.95.75.23 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_23 }}' + +[all:vars] +INVENTORY_NAME=stage +K8S_CONTROL_PLANE_ENDPOINT=194.95.75.21 +K8S_CONTROL_PLANE_ALIAS=svko-k8s-test01 + +[notebooks_gesis_org] +svko-css-backup-node + +[kubernetes_control_panel] +svko-k8s-test01 + +[kubernetes_workers] +#svko-ilcm04 +svko-css-backup-node +svko-k8s-test02 +svko-k8s-test03 + +[ingress] +svko-css-backup-node + +[harbor] +svko-css-backup-node + +[binderhub] +svko-k8s-test02 + +[jupyterhub_single_user] +svko-k8s-test03 + +[prometheus] +svko-css-backup-node + +[grafana] +svko-css-backup-node From 2eee2cb6f9fedf6bb843c01b806b71d0fe821238 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 08:19:08 +0200 Subject: [PATCH 002/128] Add k8s-common role --- .../files/etc/containerd/config.toml | 250 ++++++++++++++++++ ansible/roles/k8s-common/tasks/main.yml | 164 ++++++++++++ 2 files changed, 414 insertions(+) create mode 100644 ansible/roles/k8s-common/files/etc/containerd/config.toml create mode 100644 ansible/roles/k8s-common/tasks/main.yml diff --git a/ansible/roles/k8s-common/files/etc/containerd/config.toml b/ansible/roles/k8s-common/files/etc/containerd/config.toml new file mode 100644 index 000000000..320b460aa --- /dev/null +++ b/ansible/roles/k8s-common/files/etc/containerd/config.toml @@ -0,0 +1,250 @@ +disabled_plugins = [] +imports = [] +oom_score = 0 +plugin_dir = "" +required_plugins = [] +root = "/orc2_data/containerd" +state = "/run/containerd" +temp = "" +version = 2 + +[cgroup] + path = "" + +[debug] + address = "" + format = "" + gid = 0 + level = "" + uid = 0 + +[grpc] + address = "/run/containerd/containerd.sock" + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + tcp_address = "" + tcp_tls_ca = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 + +[metrics] + address = "" + grpc_histogram = false + +[plugins] + + [plugins."io.containerd.gc.v1.scheduler"] + deletion_threshold = 0 + mutation_threshold = 100 + pause_threshold = 0.02 + schedule_delay = "0s" + startup_delay = "100ms" + + [plugins."io.containerd.grpc.v1.cri"] + device_ownership_from_security_context = false + disable_apparmor = false + disable_cgroup = false + disable_hugetlb_controller = true + disable_proc_mount = false + disable_tcp_service = true + enable_selinux = false + enable_tls_streaming = false + enable_unprivileged_icmp = false + enable_unprivileged_ports = false + ignore_image_defined_volumes = false + max_concurrent_downloads = 3 + max_container_log_line_size = 16384 + netns_mounts_under_state_dir = false + restrict_oom_score_adj = false + sandbox_image = "registry.k8s.io/pause:3.6" + selinux_category_range = 1024 + stats_collect_period = 10 + stream_idle_timeout = "4h0m0s" + stream_server_address = "127.0.0.1" + stream_server_port = "0" + systemd_cgroup = false + tolerate_missing_hugetlb_controller = true + unset_seccomp_profile = "" + + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "" + ip_pref = "" + max_conf_num = 1 + + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + disable_snapshot_annotations = true + discard_unpacked_layers = false + ignore_rdt_not_enabled_errors = false + no_pivot = false + snapshotter = "overlayfs" + + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "" + CriuImagePath = "" + CriuPath = "" + CriuWorkPath = "" + IoGid = 0 + IoUid = 0 + NoNewKeyring = false + NoPivotRoot = false + Root = "" + ShimCgroup = "" + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options] + + [plugins."io.containerd.grpc.v1.cri".image_decryption] + key_model = "node" + + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "" + + [plugins."io.containerd.grpc.v1.cri".registry.auths] + + [plugins."io.containerd.grpc.v1.cri".registry.configs] + + [plugins."io.containerd.grpc.v1.cri".registry.headers] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" + + [plugins."io.containerd.internal.v1.restart"] + interval = "10s" + + [plugins."io.containerd.internal.v1.tracing"] + sampling_ratio = 1.0 + service_name = "containerd" + + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + + [plugins."io.containerd.runtime.v1.linux"] + no_shim = false + runtime = "runc" + runtime_root = "" + shim = "containerd-shim" + shim_debug = false + + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + sched_core = false + + [plugins."io.containerd.service.v1.diff-service"] + default = ["walking"] + + [plugins."io.containerd.service.v1.tasks-service"] + rdt_config_file = "" + + [plugins."io.containerd.snapshotter.v1.aufs"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.btrfs"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.devmapper"] + async_remove = false + base_image_size = "" + discard_blocks = false + fs_options = "" + fs_type = "" + pool_name = "" + root_path = "" + + [plugins."io.containerd.snapshotter.v1.native"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.overlayfs"] + root_path = "" + upperdir_label = false + + [plugins."io.containerd.snapshotter.v1.zfs"] + root_path = "" + + [plugins."io.containerd.tracing.processor.v1.otlp"] + endpoint = "" + insecure = false + protocol = "" + +[proxy_plugins] + +[stream_processors] + + [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"] + accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"] + args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] + env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] + path = "ctd-decoder" + returns = "application/vnd.oci.image.layer.v1.tar" + + [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"] + accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"] + args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] + env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] + path = "ctd-decoder" + returns = "application/vnd.oci.image.layer.v1.tar+gzip" + +[timeouts] + "io.containerd.timeout.bolt.open" = "0s" + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[ttrpc] + address = "" + gid = 0 + uid = 0 diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml new file mode 100644 index 000000000..7dcc68873 --- /dev/null +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -0,0 +1,164 @@ +- name: Create directory /etc/apt/keyrings if it does not exist + ansible.builtin.file: + state: directory + path: /etc/apt/keyrings +- name: Remove old Kubernetes public GPG key + ansible.builtin.file: + path: /etc/apt/trusted.gpg.d/kubernetes-archive-keyring.gpg + state: absent +- name: Remove old Kubernetes public GPG key + ansible.builtin.file: + path: /etc/apt/trusted.gpg.d/kubernetes-archive-keyring.asc + state: absent +- name: Remove old Kubernetes repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" + filename: kubernetes + state: absent +- name: Remove old Kubernetes repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes.asc] https://pkgs.k8s.io/core:/stable:/v1.27/deb/ /" + filename: kubernetes + state: absent +- name: Ensure DOCKER_CLIENT_TIMEOUT is set + ansible.builtin.lineinfile: + path: /etc/environment + regexp: '^DOCKER_CLIENT_TIMEOUT=' + line: DOCKER_CLIENT_TIMEOUT=180 +- name: Disable SWAP since kubernetes can't work with swap enabled + ansible.builtin.shell: swapoff -a +- name: Disable SWAP in fstab since kubernetes can't work with swap enabled + replace: + path: /etc/fstab + regexp: '^([^#].*?\sswap\s+sw\s+.*)$' + replace: '# \1' +- name: Disable Firewall + ansible.builtin.shell: ufw disable +- name: Allow IP forward + ansible.posix.sysctl: + name: net.ipv4.ip_forward + value: '1' + state: present +- ansible.posix.sysctl: + name: fs.inotify.max_user_instances + value: '1280' + state: present +- ansible.posix.sysctl: + name: fs.inotify.max_user_watches + value: '655360' + state: present +- name: Create /orc2_data/containerd directory if it does not exist + ansible.builtin.file: + path: /orc2_data/containerd + state: directory +- name: Create /orc2_data/repo2docker directory if it does not exist + ansible.builtin.file: + path: /orc2_data/repo2docker + state: directory +- name: Create /orc2_data/prometheus directory if it does not exist + ansible.builtin.file: + path: /orc2_data/prometheus + state: directory +- name: Create /orc2_data/grafana directory if it does not exist + ansible.builtin.file: + path: /orc2_data/grafana + state: directory +- name: Create /orc2_data/alertmanager directory if it does not exist + ansible.builtin.file: + path: /orc2_data/alertmanager + state: directory +- name: Create /harbor/jobservice directory if it does not exist + ansible.builtin.file: + path: /harbor/jobservice + state: directory +- name: Create /harbor/registry directory if it does not exist + ansible.builtin.file: + path: /harbor/registry + state: directory +- name: Create /harbor/redis directory if it does not exist + ansible.builtin.file: + path: /harbor/redis + state: directory +- name: Create /harbor/trivy directory if it does not exist + ansible.builtin.file: + path: /harbor/trivy + state: directory +- name: Create /harbor/database directory if it does not exist + ansible.builtin.file: + path: /harbor/database + state: directory +- name: Add Docker public GPG key + ansible.builtin.get_url: + url: https://download.docker.com/linux/ubuntu/gpg + dest: /etc/apt/trusted.gpg.d/docker.asc + mode: '0644' + force: true +- name: Add Docker repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/trusted.gpg.d/docker.asc] https://download.docker.com/linux/ubuntu jammy stable" + filename: docker + state: present +- name: Download Kubernetes public GPG key + ansible.builtin.get_url: + url: https://pkgs.k8s.io/core:/stable:/v1.27/deb/Release.key + dest: /tmp/kubernetes-archive-keyring.asc + mode: '0644' + force: true +- name: Convert the public GPG key to binary + ansible.builtin.command: + argv: + - gpg + - --yes + - --dearmor + - --output + - /tmp/kubernetes.gpg + - /tmp/kubernetes-archive-keyring.asc +- name: Copy GPG key + ansible.builtin.copy: + src: /tmp/kubernetes.gpg + dest: /etc/apt/keyrings/kubernetes.gpg + remote_src: true + mode: '0644' +- name: Add Kubernetes repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/keyrings/kubernetes.gpg] https://pkgs.k8s.io/core:/stable:/v1.27/deb/ /" + filename: kubernetes + state: present +- name: Add Helm public GPG key + ansible.builtin.get_url: + url: https://baltocdn.com/helm/signing.asc + dest: /etc/apt/trusted.gpg.d/helm.asc + mode: '0644' + force: true +- name: Add Helm repository + ansible.builtin.apt_repository: + repo: "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/helm.asc] https://baltocdn.com/helm/stable/debian/ all main" + filename: kubernetes + state: present +- name: Install dependencies + ansible.builtin.apt: + update_cache: true + pkg: + - rsync + - python3 + - apt-transport-https + - ca-certificates + - curl + - containerd.io=1.7.* + - kubelet=1.28.* + - kubeadm=1.28.* + - kubectl=1.28.* + - helm=3.15.* +- name: Copy containerd configuration file + ansible.builtin.copy: + src: files/etc/containerd/config.toml + dest: /etc/containerd/config.toml +- name: Reload service containerd + ansible.builtin.systemd: + name: containerd + state: restarted +- name: Enable service containerd + ansible.builtin.systemd: + name: containerd + enabled: true + masked: false \ No newline at end of file From 0b9eee2c0d2da6d6cd3240aeeb4846f306183812 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 08:28:03 +0200 Subject: [PATCH 003/128] Add Ansible playbook for GESIS --- ansible/gesis.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 ansible/gesis.yml diff --git a/ansible/gesis.yml b/ansible/gesis.yml new file mode 100644 index 000000000..e38a206a6 --- /dev/null +++ b/ansible/gesis.yml @@ -0,0 +1,4 @@ +- hosts: all + gather_facts: false + roles: + - k8s-common \ No newline at end of file From 751696b7a34cc577974eed2d2961e8a0eb59f535 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 09:17:40 +0200 Subject: [PATCH 004/128] Add GitLab CI Kubernetes agent for stage --- .gitlab/agents/stage/config.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitlab/agents/stage/config.yaml diff --git a/.gitlab/agents/stage/config.yaml b/.gitlab/agents/stage/config.yaml new file mode 100644 index 000000000..59f3bb059 --- /dev/null +++ b/.gitlab/agents/stage/config.yaml @@ -0,0 +1,3 @@ +ci_access: + projects: + - id: methods-hub/interactive-environment \ No newline at end of file From 96e940a6a37bd8cdbc802e9f0bb10b2201147745 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 08:50:24 +0200 Subject: [PATCH 005/128] Add GitLab CI --- .gitlab-ci.yml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..7079c166c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,42 @@ +stages: + - build + - deploy-stage-ansible + - deploy-stage-helm + - test-stage + - deploy-prod-nginx + - deploy-prod-helm + +.manual-web: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' + when: manual + allow_failure: true + +include: + - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.4 + inputs: + stage: build + dir: ansible + rules: + - !reference [.manual-web, rules] + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + - ansible/**/* + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == 'gesis' + + - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-deploy@10.2.4 + inputs: + stage: deploy-stage-ansible + dir: ansible + inventory: gesis-stage + playbook: gesis.yml + ssh-user: ansible + ssh-key-type: rsa + rules: + - !reference [.manual-web, rules] + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + - ansible/**/* + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == 'gesis' \ No newline at end of file From 43bec3a4c4d126493bd822de412b08b856a7d34a Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 08:51:14 +0200 Subject: [PATCH 006/128] Add Ansible vault --- ansible/vault/gesis-stage.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 ansible/vault/gesis-stage.yml diff --git a/ansible/vault/gesis-stage.yml b/ansible/vault/gesis-stage.yml new file mode 100644 index 000000000..3e7d6c8d1 --- /dev/null +++ b/ansible/vault/gesis-stage.yml @@ -0,0 +1,22 @@ +$ANSIBLE_VAULT;1.1;AES256 +34323935313236336637636634363561333735623162386531343764313534333565646338376137 +6430636362386233343430343866316439633737666332360a666531383465613037323262653230 +34303435303833383464353963353035653430343263636664663534333139666531336264353161 +3838613736666339630a653531643666626432323661376639346230666434633733356237653330 +64323138343831383635353337353132623435656430383038303034373033653863323262343033 +32353864393761636634323434333262363839323463316563316631393433656265663465656263 +37653633303231633534633837626531373537663961636638333365383865303836633733323635 +30336561343562633861313862343737353365373435363363303730343464636265383366323730 +63663265356466336336336233616434373733623536646130373837303864396663636665373635 +66613932323262356333346164656231343130633765343236316235626239303538383963346362 +35316338616532353862316131323264653562356361343930323938653366393962663935653239 +64653037376465623665343365643337386664313431393833333937313635303363333035333235 +37353731366162383661636132646466333361366638373836636261336239386232623538333237 +35646433666465643966623331346434653937316531643764393331316366363663313736393636 +35396562386563626132306435383763626163343965343133646532323932613065373935306562 +64376130383733343261323033303863643063393233383438663566376466663465623136396164 +65336534626464633065333861303463623763353562666539663331326562336139323532663364 +62646633346536626564643032613265613962643935613035306632333638333634303436663939 +32663539656530366133663063643230346165633038303738623264646166313762646432323732 +32623262353338313232343732336536366238666631323634663930353534396133353439363630 +39643938373262313934626163313235303866313537353166633432666131366134 From ce9454c3a8ba21f4ae1568b9bbf4e4154931b1c4 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 09:25:49 +0200 Subject: [PATCH 007/128] Remove manual option for GitLab CI --- .gitlab-ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7079c166c..ea91cb2c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,19 +6,12 @@ stages: - deploy-prod-nginx - deploy-prod-helm -.manual-web: - rules: - - if: $CI_PIPELINE_SOURCE == 'web' - when: manual - allow_failure: true - include: - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.4 inputs: stage: build dir: ansible rules: - - !reference [.manual-web, rules] - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: - ansible/**/* @@ -34,7 +27,6 @@ include: ssh-user: ansible ssh-key-type: rsa rules: - - !reference [.manual-web, rules] - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: - ansible/**/* From 9944c78cbc00c23970b9aa22bc35719481f80cd7 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 09:28:15 +0200 Subject: [PATCH 008/128] Remove ssh-key-type from GitLab CI --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ea91cb2c2..21423bdf8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,6 @@ include: inventory: gesis-stage playbook: gesis.yml ssh-user: ansible - ssh-key-type: rsa rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: From 239265c15825dbc68b60f36454a540bee25e73db Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 09:34:20 +0200 Subject: [PATCH 009/128] Upgrade Ansible Component --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21423bdf8..79b689017 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ stages: - deploy-prod-helm include: - - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.4 + - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.5 inputs: stage: build dir: ansible @@ -18,7 +18,7 @@ include: - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == 'gesis' - - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-deploy@10.2.4 + - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-deploy@10.2.5 inputs: stage: deploy-stage-ansible dir: ansible From 12ae72e30a1f3b3030e26700f6f06b2182fa76e6 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 10:00:15 +0200 Subject: [PATCH 010/128] Upgrade Ansible Component --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79b689017..9d1c4ae23 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ stages: - deploy-prod-helm include: - - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.5 + - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.6 inputs: stage: build dir: ansible @@ -18,7 +18,7 @@ include: - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == 'gesis' - - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-deploy@10.2.5 + - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-deploy@10.2.6 inputs: stage: deploy-stage-ansible dir: ansible From f58e1c82fd823076b212f5893a8cc586e976982c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 10:08:38 +0200 Subject: [PATCH 011/128] Add smoke test --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d1c4ae23..b9c4744d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,4 +30,9 @@ include: changes: - ansible/**/* - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == 'gesis' \ No newline at end of file + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == 'gesis' + +smoke test to stage cluster: + stage: test-stage + script: + - curl https://notebooks-test.gesis.org/binder/ \ No newline at end of file From 60484192964c2596c3ab33f9f7f7f181fdeb1e47 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 10:17:25 +0200 Subject: [PATCH 012/128] Add EditorConfig --- .editorconfig | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..216de0c51 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[ansible/**] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = space +insert_final_newline = true \ No newline at end of file From abe6e301d361d2d607a5bdd0c002498c9e0f4f9f Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 10:30:35 +0200 Subject: [PATCH 013/128] Fix warming from Ansible --- ansible/gesis.yml | 5 ++- ansible/roles/k8s-common/tasks/main.yml | 53 ++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/ansible/gesis.yml b/ansible/gesis.yml index e38a206a6..8493aa813 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -1,4 +1,5 @@ -- hosts: all +- name: Configure servers that are part of Kubernetes cluster + hosts: all gather_facts: false roles: - - k8s-common \ No newline at end of file + - k8s-common diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index 7dcc68873..2e966fd4f 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -2,6 +2,9 @@ ansible.builtin.file: state: directory path: /etc/apt/keyrings + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Remove old Kubernetes public GPG key ansible.builtin.file: path: /etc/apt/trusted.gpg.d/kubernetes-archive-keyring.gpg @@ -26,24 +29,28 @@ regexp: '^DOCKER_CLIENT_TIMEOUT=' line: DOCKER_CLIENT_TIMEOUT=180 - name: Disable SWAP since kubernetes can't work with swap enabled - ansible.builtin.shell: swapoff -a + ansible.builtin.command: swapoff -a + changed_when: false - name: Disable SWAP in fstab since kubernetes can't work with swap enabled - replace: + ansible.builtin.replace: path: /etc/fstab regexp: '^([^#].*?\sswap\s+sw\s+.*)$' replace: '# \1' - name: Disable Firewall - ansible.builtin.shell: ufw disable + ansible.builtin.command: ufw disable + changed_when: false - name: Allow IP forward ansible.posix.sysctl: name: net.ipv4.ip_forward value: '1' state: present -- ansible.posix.sysctl: +- name: Set inotify max user instances + ansible.posix.sysctl: name: fs.inotify.max_user_instances value: '1280' state: present -- ansible.posix.sysctl: +- name: Set inotify max user watches + ansible.posix.sysctl: name: fs.inotify.max_user_watches value: '655360' state: present @@ -51,42 +58,72 @@ ansible.builtin.file: path: /orc2_data/containerd state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Create /orc2_data/repo2docker directory if it does not exist ansible.builtin.file: path: /orc2_data/repo2docker state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Create /orc2_data/prometheus directory if it does not exist ansible.builtin.file: path: /orc2_data/prometheus state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Create /orc2_data/grafana directory if it does not exist ansible.builtin.file: path: /orc2_data/grafana state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Create /orc2_data/alertmanager directory if it does not exist ansible.builtin.file: path: /orc2_data/alertmanager state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Create /harbor/jobservice directory if it does not exist ansible.builtin.file: path: /harbor/jobservice state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Create /harbor/registry directory if it does not exist ansible.builtin.file: path: /harbor/registry state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Create /harbor/redis directory if it does not exist ansible.builtin.file: path: /harbor/redis state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Create /harbor/trivy directory if it does not exist ansible.builtin.file: path: /harbor/trivy state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Create /harbor/database directory if it does not exist ansible.builtin.file: path: /harbor/database state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Add Docker public GPG key ansible.builtin.get_url: url: https://download.docker.com/linux/ubuntu/gpg @@ -113,6 +150,7 @@ - --output - /tmp/kubernetes.gpg - /tmp/kubernetes-archive-keyring.asc + changed_when: false - name: Copy GPG key ansible.builtin.copy: src: /tmp/kubernetes.gpg @@ -153,6 +191,9 @@ ansible.builtin.copy: src: files/etc/containerd/config.toml dest: /etc/containerd/config.toml + owner: root + group: root + mode: u=rw,g=r,o=r - name: Reload service containerd ansible.builtin.systemd: name: containerd @@ -161,4 +202,4 @@ ansible.builtin.systemd: name: containerd enabled: true - masked: false \ No newline at end of file + masked: false From 0db8dfea88fb0e283d9a3507a6c669c264b3a438 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 10:12:43 +0200 Subject: [PATCH 014/128] Remove rules for debug --- .gitlab-ci.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9c4744d0..41c2fe645 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,12 +11,6 @@ include: inputs: stage: build dir: ansible - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - changes: - - ansible/**/* - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == 'gesis' - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-deploy@10.2.6 inputs: @@ -25,12 +19,6 @@ include: inventory: gesis-stage playbook: gesis.yml ssh-user: ansible - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - changes: - - ansible/**/* - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == 'gesis' smoke test to stage cluster: stage: test-stage From 7633776286b9e176d8dc74772404374f1e07a361 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 11:45:02 +0200 Subject: [PATCH 015/128] Fix Ansible Vault --- ansible/vault/gesis-stage.yml | 42 +++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/ansible/vault/gesis-stage.yml b/ansible/vault/gesis-stage.yml index 3e7d6c8d1..c6ce4e92c 100644 --- a/ansible/vault/gesis-stage.yml +++ b/ansible/vault/gesis-stage.yml @@ -1,22 +1,22 @@ $ANSIBLE_VAULT;1.1;AES256 -34323935313236336637636634363561333735623162386531343764313534333565646338376137 -6430636362386233343430343866316439633737666332360a666531383465613037323262653230 -34303435303833383464353963353035653430343263636664663534333139666531336264353161 -3838613736666339630a653531643666626432323661376639346230666434633733356237653330 -64323138343831383635353337353132623435656430383038303034373033653863323262343033 -32353864393761636634323434333262363839323463316563316631393433656265663465656263 -37653633303231633534633837626531373537663961636638333365383865303836633733323635 -30336561343562633861313862343737353365373435363363303730343464636265383366323730 -63663265356466336336336233616434373733623536646130373837303864396663636665373635 -66613932323262356333346164656231343130633765343236316235626239303538383963346362 -35316338616532353862316131323264653562356361343930323938653366393962663935653239 -64653037376465623665343365643337386664313431393833333937313635303363333035333235 -37353731366162383661636132646466333361366638373836636261336239386232623538333237 -35646433666465643966623331346434653937316531643764393331316366363663313736393636 -35396562386563626132306435383763626163343965343133646532323932613065373935306562 -64376130383733343261323033303863643063393233383438663566376466663465623136396164 -65336534626464633065333861303463623763353562666539663331326562336139323532663364 -62646633346536626564643032613265613962643935613035306632333638333634303436663939 -32663539656530366133663063643230346165633038303738623264646166313762646432323732 -32623262353338313232343732336536366238666631323634663930353534396133353439363630 -39643938373262313934626163313235303866313537353166633432666131366134 +65666231316164316637653330376337383937373938613334343066376139326661643962376237 +3739366536353237356539656138383164326139333139390a333134313565323232646639313162 +61656433306461343266393566626465316239353933303136633034343231666337363838623563 +6633633234626132390a333632353730353066326438623663383634343532333539366363333334 +34646163313065393732306363353231633239313637646339623032626366626436346234376130 +66636432383138383838616434303931316334386665303563376336623930356638666366333561 +66353830353361343335623737653130383862353638393336303866323738303865623934303830 +66663164353837626636653766646233666164393564396233656665646636643862643035383733 +65376535346438623032316666333265643135653035373139626232646430623733383134656533 +34323737613565663536643430613832636666653030383066316632336363323734326339376162 +39343665393661623530303236353165656130396137373634363265346362623832653563613338 +31313261646333656362636134306162666133373334653933366531643063643537353663353932 +39386538626664393536363035646265643832303961323636653037356433346266353963666164 +32653334653936633130316463303061343938363630376663613639636338343331353732363837 +37616137373834333836393137333131643432653239313432623462616537353337303432393736 +34333463636566373330346437653037313366633762623161616564376639376561333561366530 +37356235373336303563373137393263626532356333666166396435346565333964316263393665 +32636239396563326635363636396435623731613364376632336261643064336530616235386631 +37336230323331323838326331303831616337363833616563306131393733666663303836636366 +38656336373763353836643536376239316463353862323332626661346366636236613530366464 +36363832656263633161303335613332396237353865643964626462653565386562 From 36b439df09c19d4e762c37476e44a39c26f3c053 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 11:48:26 +0200 Subject: [PATCH 016/128] Fix SSH key type --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41c2fe645..70ce2d518 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,6 +19,7 @@ include: inventory: gesis-stage playbook: gesis.yml ssh-user: ansible + ssh-key-type: ed25519 smoke test to stage cluster: stage: test-stage From b279cdf156264a752f60350c5c5ed55274780b8e Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 11:52:41 +0200 Subject: [PATCH 017/128] Enable root for Ansible --- ansible/gesis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/gesis.yml b/ansible/gesis.yml index 8493aa813..e8484cf52 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -1,5 +1,6 @@ - name: Configure servers that are part of Kubernetes cluster hosts: all gather_facts: false + become: true roles: - k8s-common From e798169001653ddda0abe71bf06be0f164187475 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 12:00:09 +0200 Subject: [PATCH 018/128] Configure Kubernetes control panel --- ansible/gesis.yml | 6 ++ .../roles/k8s-control-panel/tasks/main.yml | 60 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 ansible/roles/k8s-control-panel/tasks/main.yml diff --git a/ansible/gesis.yml b/ansible/gesis.yml index e8484cf52..f8c06cd3a 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -4,3 +4,9 @@ become: true roles: - k8s-common +- name: Configure Kubernetes control panel + hosts: kubernetes_control_panel + gather_facts: false + become: true + roles: + - k8s-control-panel diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml new file mode 100644 index 000000000..0537dcd9d --- /dev/null +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -0,0 +1,60 @@ +- name: Check if Kubernetes is running + ansible.builtin.shell: > + kubectl get nodes || /bin/true + register: kubernetes_nodes +- name: Pull kubernetes images + when: kubernetes_nodes.stdout.find('control-plane') == -1 + ansible.builtin.shell: > + kubeadm config images pull + --cri-socket unix:///run/containerd/containerd.sock +- name: Initialize the cluster + when: kubernetes_nodes.stdout.find('control-plane') == -1 + ansible.builtin.shell: > + kubeadm init + --pod-network-cidr=10.244.0.0/16 + --upload-certs + --control-plane-endpoint={{ K8S_CONTROL_PLANE_ENDPOINT }} + --cri-socket unix:///run/containerd/containerd.sock + register: kubeadm_init_output +- name: Create root's .kube directory + ansible.builtin.file: + path: /root/.kube + state: directory + mode: 0755 +- name: Copies admin.conf to root's kube config + ansible.builtin.copy: + src: /etc/kubernetes/admin.conf + dest: /root/.kube/config + remote_src: true +- name: Create user's .kube directory + ansible.builtin.file: + path: /home/ansible/.kube + state: directory + mode: 0755 + owner: ansible + group: ansible +- name: Copies admin.conf to user's kube config + ansible.builtin.copy: + src: /etc/kubernetes/admin.conf + dest: /home/ansible/.kube/config + remote_src: true + owner: ansible + group: ansible +- name: Get the token for joining the worker nodes + ansible.builtin.shell: > + kubeadm token create --print-join-command + register: kubernetes_join_command +- name: Create temporary file + ansible.builtin.file: + path: /tmp/kubernetes_join_command + state: touch + owner: ansible + group: ansible +- name: Save content of join command + ansible.builtin.copy: + content: "{{ kubernetes_join_command.stdout }}" + dest: /tmp/kubernetes_join_command +- name: Copy join command to local file + ansible.builtin.fetch: + src: /tmp/kubernetes_join_command + dest: '{{ ANSIBLE_CONTROL_NODE_TMP }}' From ad735b2c0638933b5f839e2c354b68f779b840b5 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 12:01:57 +0200 Subject: [PATCH 019/128] Configure Kubernetes worker --- ansible/gesis.yml | 6 ++++++ ansible/roles/k8s-worker/tasks/main.yml | 11 +++++++++++ 2 files changed, 17 insertions(+) create mode 100644 ansible/roles/k8s-worker/tasks/main.yml diff --git a/ansible/gesis.yml b/ansible/gesis.yml index f8c06cd3a..3439b4df3 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -10,3 +10,9 @@ become: true roles: - k8s-control-panel +- name: Configure Kubernetes workers + hosts: kubernetes_workers + gather_facts: false + become: true + roles: + - k8s-worker diff --git a/ansible/roles/k8s-worker/tasks/main.yml b/ansible/roles/k8s-worker/tasks/main.yml new file mode 100644 index 000000000..e78d7f86b --- /dev/null +++ b/ansible/roles/k8s-worker/tasks/main.yml @@ -0,0 +1,11 @@ +- name: Copy join command + ansible.builtin.copy: + src: '{{ ANSIBLE_CONTROL_NODE_TMP }}/{{ K8S_CONTROL_PLANE_ALIAS }}/tmp/kubernetes_join_command' + dest: /tmp/kubernetes_join_command + mode: u=rwx,g=rx,o=rx +- name: Attempt to join cluster + ansible.builtin.shell: /tmp/kubernetes_join_command + register: kubernetes_join_attempt + failed_when: + - kubernetes_join_attempt.rc != 0 + - '"already exists" not in kubernetes_join_attempt.stderr' From b94c01134ef562a7f7038d179d9ff2a7f08f7d70 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 12:04:38 +0200 Subject: [PATCH 020/128] Configure kernels modules at boot --- ansible/roles/k8s-common/tasks/main.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index 2e966fd4f..6e321253e 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -203,3 +203,17 @@ name: containerd enabled: true masked: false +- name: modprobe overlay + ansible.builtin.shell: modprobe overlay +- name: modprobe br_netfilter + ansible.builtin.shell: modprobe br_netfilter +- name: Create file for list of kernel modules required by containerd + ansible.builtin.file: + path: "/etc/modules-load.d/containerd.conf" + state: "touch" +- name: Populate list of kernel modules required by containerd + ansible.builtin.blockinfile: + path: "/etc/modules-load.d/containerd.conf" + block: | + overlay + br_netfilter From dc19c9b72e91986b0d6600da4ca1045ea9e71add Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 12:07:17 +0200 Subject: [PATCH 021/128] Configure Calico --- .../files/custom-resources.yaml | 24 +++++++++++++++++++ .../roles/k8s-control-panel/tasks/main.yml | 12 ++++++++++ 2 files changed, 36 insertions(+) create mode 100644 ansible/roles/k8s-control-panel/files/custom-resources.yaml diff --git a/ansible/roles/k8s-control-panel/files/custom-resources.yaml b/ansible/roles/k8s-control-panel/files/custom-resources.yaml new file mode 100644 index 000000000..4a90b5bc2 --- /dev/null +++ b/ansible/roles/k8s-control-panel/files/custom-resources.yaml @@ -0,0 +1,24 @@ +# This section includes base Calico installation configuration. +# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + # Configures Calico networking. + calicoNetwork: + # Note: The ipPools section cannot be modified post-install. + ipPools: + - blockSize: 26 + cidr: 10.244.0.0/16 + encapsulation: VXLANCrossSubnet + natOutgoing: Enabled + nodeSelector: all() +--- +# This section configures the Calico API server. +# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer +apiVersion: operator.tigera.io/v1 +kind: APIServer +metadata: + name: default +spec: {} diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 0537dcd9d..e80aa138f 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -58,3 +58,15 @@ ansible.builtin.fetch: src: /tmp/kubernetes_join_command dest: '{{ ANSIBLE_CONTROL_NODE_TMP }}' +- name: Remove Container Network Interface (CNI) Flannel + kubernetes.core.k8s: + state: absent + src: https://github.com/coreos/flannel/raw/master/Documentation/kube-flannel.yml +- name: Install Container Network Interface (CNI) Tigera Calico operator + kubernetes.core.k8s: + state: present + src: https://raw.githubusercontent.com/projectcalico/calico/v3.26.3/manifests/tigera-operator.yaml +- name: Install Calico and resource + kubernetes.core.k8s: + state: present + definition: "{{ lookup('file', '../calico/custom-resources.yaml') | from_yaml_all }}" From 53aa48e1b30abb1b44e8a34d02f12d5128bd667c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 12:10:14 +0200 Subject: [PATCH 022/128] Configure JupyterHub worker --- ansible/gesis.yml | 6 +++ .../files/var/lib/kubelet/config.yaml | 45 +++++++++++++++++++ ansible/roles/jupyterhub/tasks/main.yml | 12 +++++ 3 files changed, 63 insertions(+) create mode 100644 ansible/roles/jupyterhub/files/var/lib/kubelet/config.yaml create mode 100644 ansible/roles/jupyterhub/tasks/main.yml diff --git a/ansible/gesis.yml b/ansible/gesis.yml index 3439b4df3..b470e45aa 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -16,3 +16,9 @@ become: true roles: - k8s-worker +- name: Configure JupyterHub workers + hosts: jupyterhub_single_user + gather_facts: false + become: true + roles: + - k8s-worker diff --git a/ansible/roles/jupyterhub/files/var/lib/kubelet/config.yaml b/ansible/roles/jupyterhub/files/var/lib/kubelet/config.yaml new file mode 100644 index 000000000..cbe083dae --- /dev/null +++ b/ansible/roles/jupyterhub/files/var/lib/kubelet/config.yaml @@ -0,0 +1,45 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s +cgroupDriver: systemd +clusterDNS: + - 10.96.0.10 +clusterDomain: cluster.local +cpuManagerReconcilePeriod: 0s +evictionPressureTransitionPeriod: 0s +fileCheckFrequency: 0s +healthzBindAddress: 127.0.0.1 +healthzPort: 10248 +httpCheckFrequency: 0s +imageMinimumGCAge: 0s +kind: KubeletConfiguration +logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 +memorySwap: {} +nodeStatusReportFrequency: 0s +nodeStatusUpdateFrequency: 0s +resolvConf: /run/systemd/resolve/resolv.conf +rotateCertificates: true +runtimeRequestTimeout: 0s +shutdownGracePeriod: 0s +shutdownGracePeriodCriticalPods: 0s +staticPodPath: /etc/kubernetes/manifests +streamingConnectionIdleTimeout: 0s +syncFrequency: 0s +volumeStatsAggPeriod: 0s +maxPods: 500 diff --git a/ansible/roles/jupyterhub/tasks/main.yml b/ansible/roles/jupyterhub/tasks/main.yml new file mode 100644 index 000000000..288105288 --- /dev/null +++ b/ansible/roles/jupyterhub/tasks/main.yml @@ -0,0 +1,12 @@ +- name: Stop kubelet service + ansible.builtin.systemd: + name: kubelet + state: stopped +- name: Copy kubelet configuration + ansible.builtin.copy: + src: ../var/lib/kubelet/config.yaml + dest: /var/lib/kubelet/config.yaml +- name: Restarted kubelet service + ansible.builtin.systemd: + name: kubelet + state: restarted From 8dbd358081bbfe0fec3d8180ef69b194447bd78e Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 12:17:57 +0200 Subject: [PATCH 023/128] Add labels to Kubernetes nodes --- ansible/gesis.yml | 5 +++++ ansible/roles/mybinder/tasks/main.yml | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 ansible/roles/mybinder/tasks/main.yml diff --git a/ansible/gesis.yml b/ansible/gesis.yml index b470e45aa..66b92df7d 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -22,3 +22,8 @@ become: true roles: - k8s-worker +- name: Configure mybinder.org Kubernetes cluster + hosts: kubernetes_control_panel + gather_facts: false + roles: + - mybinder diff --git a/ansible/roles/mybinder/tasks/main.yml b/ansible/roles/mybinder/tasks/main.yml new file mode 100644 index 000000000..f4b164986 --- /dev/null +++ b/ansible/roles/mybinder/tasks/main.yml @@ -0,0 +1,20 @@ +- name: Add hub.jupyter.org/node-purpose label + ansible.builtin.shell: | + {% for host in hostvars %} + {% if host in groups['binderhub'] %} + kubectl label nodes {{ host }} hub.jupyter.org/node-purpose=core + {% else %} + kubectl label nodes {{ host }} hub.jupyter.org/node-purpose- + {% endif %} + {% endfor %} +- name: Add labels from inventory + ansible.builtin.shell: | + {% for host in hostvars %} + {% for group, host_list in groups.items() %} + {% if host in host_list %} + kubectl label nodes {{ host }} {{ group }}=true + {% else %} + kubectl label nodes {{ host }} {{ group }}- + {% endif %} + {% endfor %} + {% endfor %} From dbe21b428db64da9d12393034a43ce7dc0be4c6b Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 12:29:55 +0200 Subject: [PATCH 024/128] Fix Ansible warnings --- ansible/roles/jupyterhub/tasks/main.yml | 3 +++ ansible/roles/k8s-common/tasks/main.yml | 13 +++++++++---- ansible/roles/k8s-control-panel/tasks/main.yml | 18 ++++++++++++++++-- ansible/roles/k8s-worker/tasks/main.yml | 3 ++- ansible/roles/mybinder/tasks/main.yml | 2 ++ 5 files changed, 32 insertions(+), 7 deletions(-) diff --git a/ansible/roles/jupyterhub/tasks/main.yml b/ansible/roles/jupyterhub/tasks/main.yml index 288105288..1409bf70d 100644 --- a/ansible/roles/jupyterhub/tasks/main.yml +++ b/ansible/roles/jupyterhub/tasks/main.yml @@ -6,6 +6,9 @@ ansible.builtin.copy: src: ../var/lib/kubelet/config.yaml dest: /var/lib/kubelet/config.yaml + owner: root + group: root + mode: u=rw,g=r,o=r - name: Restarted kubelet service ansible.builtin.systemd: name: kubelet diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index 6e321253e..e4f72ee71 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -203,14 +203,19 @@ name: containerd enabled: true masked: false -- name: modprobe overlay - ansible.builtin.shell: modprobe overlay -- name: modprobe br_netfilter - ansible.builtin.shell: modprobe br_netfilter +- name: Modify kernel module overlay + ansible.builtin.command: modprobe overlay + changed_when: false +- name: Modify kernel module br_netfilter + ansible.builtin.command: modprobe br_netfilter + changed_when: false - name: Create file for list of kernel modules required by containerd ansible.builtin.file: path: "/etc/modules-load.d/containerd.conf" state: "touch" + owner: root + group: root + mode: u=rw,g=r,o=r - name: Populate list of kernel modules required by containerd ansible.builtin.blockinfile: path: "/etc/modules-load.d/containerd.conf" diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index e80aa138f..9ff1dfbe0 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -1,12 +1,14 @@ - name: Check if Kubernetes is running ansible.builtin.shell: > kubectl get nodes || /bin/true + changed_when: false register: kubernetes_nodes - name: Pull kubernetes images when: kubernetes_nodes.stdout.find('control-plane') == -1 ansible.builtin.shell: > kubeadm config images pull --cri-socket unix:///run/containerd/containerd.sock + changed_when: false - name: Initialize the cluster when: kubernetes_nodes.stdout.find('control-plane') == -1 ansible.builtin.shell: > @@ -15,22 +17,28 @@ --upload-certs --control-plane-endpoint={{ K8S_CONTROL_PLANE_ENDPOINT }} --cri-socket unix:///run/containerd/containerd.sock + changed_when: false register: kubeadm_init_output - name: Create root's .kube directory ansible.builtin.file: path: /root/.kube state: directory - mode: 0755 + owner: root + group: root + mode: u=rwx,g=rx,o=rx - name: Copies admin.conf to root's kube config ansible.builtin.copy: src: /etc/kubernetes/admin.conf dest: /root/.kube/config remote_src: true + owner: root + group: root + mode: u=rw,g=r,o= - name: Create user's .kube directory ansible.builtin.file: path: /home/ansible/.kube state: directory - mode: 0755 + mode: u=rwx,g=rx,o=rx owner: ansible group: ansible - name: Copies admin.conf to user's kube config @@ -40,9 +48,11 @@ remote_src: true owner: ansible group: ansible + mode: u=rw,g=r,o= - name: Get the token for joining the worker nodes ansible.builtin.shell: > kubeadm token create --print-join-command + changed_when: false register: kubernetes_join_command - name: Create temporary file ansible.builtin.file: @@ -50,10 +60,14 @@ state: touch owner: ansible group: ansible + mode: u=rw,g=r,o= - name: Save content of join command ansible.builtin.copy: content: "{{ kubernetes_join_command.stdout }}" dest: /tmp/kubernetes_join_command + owner: ansible + group: ansible + mode: u=rw,g=r,o= - name: Copy join command to local file ansible.builtin.fetch: src: /tmp/kubernetes_join_command diff --git a/ansible/roles/k8s-worker/tasks/main.yml b/ansible/roles/k8s-worker/tasks/main.yml index e78d7f86b..7f1d6b455 100644 --- a/ansible/roles/k8s-worker/tasks/main.yml +++ b/ansible/roles/k8s-worker/tasks/main.yml @@ -4,8 +4,9 @@ dest: /tmp/kubernetes_join_command mode: u=rwx,g=rx,o=rx - name: Attempt to join cluster - ansible.builtin.shell: /tmp/kubernetes_join_command + ansible.builtin.command: /tmp/kubernetes_join_command register: kubernetes_join_attempt failed_when: - kubernetes_join_attempt.rc != 0 - '"already exists" not in kubernetes_join_attempt.stderr' + changed_when: false diff --git a/ansible/roles/mybinder/tasks/main.yml b/ansible/roles/mybinder/tasks/main.yml index f4b164986..720cb1e31 100644 --- a/ansible/roles/mybinder/tasks/main.yml +++ b/ansible/roles/mybinder/tasks/main.yml @@ -7,6 +7,7 @@ kubectl label nodes {{ host }} hub.jupyter.org/node-purpose- {% endif %} {% endfor %} + changed_when: false - name: Add labels from inventory ansible.builtin.shell: | {% for host in hostvars %} @@ -18,3 +19,4 @@ {% endif %} {% endfor %} {% endfor %} + changed_when: false From 917c5898b7a83edf391a9688331bcc5b518f9610 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 12:34:12 +0200 Subject: [PATCH 025/128] Fix path to calico custom resources --- ansible/roles/k8s-control-panel/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 9ff1dfbe0..65cc24dda 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -83,4 +83,4 @@ - name: Install Calico and resource kubernetes.core.k8s: state: present - definition: "{{ lookup('file', '../calico/custom-resources.yaml') | from_yaml_all }}" + definition: "{{ lookup('file', 'files/calico/custom-resources.yaml') | from_yaml_all }}" From ec4f3bfe86d0a6de4a24ce854975a7c19a2d5f58 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 13:05:38 +0200 Subject: [PATCH 026/128] Fix loopup based on ChatGPT answer --- ansible/roles/k8s-control-panel/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 65cc24dda..66329daee 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -83,4 +83,4 @@ - name: Install Calico and resource kubernetes.core.k8s: state: present - definition: "{{ lookup('file', 'files/calico/custom-resources.yaml') | from_yaml_all }}" + definition: "{{ lookup('file', '{{ role_path }}files/calico/custom-resources.yaml') | from_yaml_all }}" From bcc2690ee8dcb92cf92663bfe89d01de7d0dd8f6 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 13:11:33 +0200 Subject: [PATCH 027/128] Fix missing / in path --- ansible/roles/k8s-control-panel/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 66329daee..04b310842 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -83,4 +83,4 @@ - name: Install Calico and resource kubernetes.core.k8s: state: present - definition: "{{ lookup('file', '{{ role_path }}files/calico/custom-resources.yaml') | from_yaml_all }}" + definition: "{{ lookup('file', '{{ role_path }}/files/calico/custom-resources.yaml') | from_yaml_all }}" From d9212da4a92cc6aca593ab3a21c595d2ce0e3917 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 13:13:25 +0200 Subject: [PATCH 028/128] Fix file location --- .../k8s-control-panel/files/{ => calico}/custom-resources.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ansible/roles/k8s-control-panel/files/{ => calico}/custom-resources.yaml (100%) diff --git a/ansible/roles/k8s-control-panel/files/custom-resources.yaml b/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml similarity index 100% rename from ansible/roles/k8s-control-panel/files/custom-resources.yaml rename to ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml From 789c5e69ae11d5e0840ca0215b22a0ac010a3c69 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 13:22:41 +0200 Subject: [PATCH 029/128] Add shebang to file --- ansible/roles/k8s-control-panel/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 04b310842..b1ffc2321 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -63,7 +63,9 @@ mode: u=rw,g=r,o= - name: Save content of join command ansible.builtin.copy: - content: "{{ kubernetes_join_command.stdout }}" + content: | + #!/bin/sh + {{ kubernetes_join_command.stdout }} dest: /tmp/kubernetes_join_command owner: ansible group: ansible From e5a60eda97681fc042eb030b1a7ac4404407c469 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 13:38:40 +0200 Subject: [PATCH 030/128] Add more configuration to Kubernetes control node --- ansible/roles/k8s-common/tasks/main.yml | 3 + .../files/cron/kill-after-timeout-pods.py | 93 +++++++++++ .../files/cron/kill-succeeded-pods.py | 79 ++++++++++ .../systemd/system/orc2-fix-dind-bot.service | 14 ++ .../system/orc2-fix-jupyterhub-bot.service | 11 ++ .../files/usr/bin/orc2-fix-dind-bot.py | 146 ++++++++++++++++++ .../files/usr/bin/orc2-fix-jupyterhub-bot.py | 111 +++++++++++++ .../roles/k8s-control-panel/tasks/main.yml | 87 +++++++++++ 8 files changed, 544 insertions(+) create mode 100644 ansible/roles/k8s-control-panel/files/cron/kill-after-timeout-pods.py create mode 100644 ansible/roles/k8s-control-panel/files/cron/kill-succeeded-pods.py create mode 100644 ansible/roles/k8s-control-panel/files/etc/systemd/system/orc2-fix-dind-bot.service create mode 100644 ansible/roles/k8s-control-panel/files/etc/systemd/system/orc2-fix-jupyterhub-bot.service create mode 100644 ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-dind-bot.py create mode 100644 ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-jupyterhub-bot.py diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index e4f72ee71..5495cf645 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -179,6 +179,9 @@ pkg: - rsync - python3 + - python3-kubernetes + - python3-invoke + - python3-fabric - apt-transport-https - ca-certificates - curl diff --git a/ansible/roles/k8s-control-panel/files/cron/kill-after-timeout-pods.py b/ansible/roles/k8s-control-panel/files/cron/kill-after-timeout-pods.py new file mode 100644 index 000000000..e52f7d828 --- /dev/null +++ b/ansible/roles/k8s-control-panel/files/cron/kill-after-timeout-pods.py @@ -0,0 +1,93 @@ +"""Kill pods in Kubernetes cluster after timeout""" + +import argparse +import logging +import datetime + +from kubernetes import client, config + +logging.basicConfig( + format="%(asctime)s %(levelname)-8s | %(message)s", datefmt="%Y-%m-%d %H:%M:%S" +) +logger = logging.getLogger("kill-after-timeout-pods") +logger.setLevel(logging.WARNING) + +NAMESPACE = "gesis" +BINDER_TIME_OUT = 6 # hours + + +def get_timed_out_pods(): + """Get list of all timed out pods that are single user running pod""" + time_now = datetime.datetime.now(datetime.timezone.utc) + all_timed_out_pods = [] + + api_response = v1.list_namespaced_pod(NAMESPACE) + for pod in api_response.items: + pod_run_time = time_now - pod.metadata.creation_timestamp + pod_run_time_in_hours = pod_run_time.total_seconds() / 3600 + logger.debug( + "Pod %s (%s) is running for %.1f hours.", + pod.metadata.name, + pod.status.phase, + pod_run_time_in_hours, + ) + if ( + pod.metadata.name.startswith("jupyter-") + and pod_run_time_in_hours > BINDER_TIME_OUT + ): + all_timed_out_pods.append(pod) + logger.debug("Pod %s added to the list.", pod.metadata.name) + + return all_timed_out_pods + + +def kill_pod(pod): + """Kill single pod""" + logger.info("Requesting delete of pod %s ...", pod.metadata.name) + try: + api_response = v1.delete_namespaced_pod(pod.metadata.name, NAMESPACE) + logger.info("Pod %s deleted.", api_response.metadata.name) + except client.exceptions.ApiException as exception: + logger.info( + "Fail to delete pod %s due %s", pod.metadata.name, exception + ) + + +def kill_timed_out_pods(): + """Kill timed out pods""" + logger.info("Starting inspection of Kubernetes pod ...") + all_timed_out_pods = get_timed_out_pods() + for timed_out_pod in all_timed_out_pods: + kill_pod(timed_out_pod) + logger.info("%s pods deleted.", len(all_timed_out_pods)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="Open Research Computing v2 Kill Timed Out Pods Cron Job", + description="Cron job to kill Kubernetes pods that timed out", + ) + parser.add_argument( + "-c", + "--kube-config", + type=str, + default="~/.kube/config", + help="Location of Kubernetes configuration file", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Display log information" + ) + parser.add_argument( + "-vv", "--debug", action="store_true", help="Display debug information" + ) + args = parser.parse_args() + if args.verbose: + logger.setLevel(logging.INFO) + if args.debug: + logger.setLevel(logging.DEBUG) + + config.load_kube_config(config_file=args.kube_config) + + v1 = client.CoreV1Api() + + kill_timed_out_pods() diff --git a/ansible/roles/k8s-control-panel/files/cron/kill-succeeded-pods.py b/ansible/roles/k8s-control-panel/files/cron/kill-succeeded-pods.py new file mode 100644 index 000000000..deb909c1d --- /dev/null +++ b/ansible/roles/k8s-control-panel/files/cron/kill-succeeded-pods.py @@ -0,0 +1,79 @@ +"""Kill succeeded pods in Kubernetes cluster""" + +import argparse +import logging + +from kubernetes import client, config + +logging.basicConfig( + format="%(asctime)s %(levelname)-8s | %(message)s", datefmt="%Y-%m-%d %H:%M:%S" +) +logger = logging.getLogger("kill-succeeded-pods") +logger.setLevel(logging.WARNING) + +NAMESPACE = "gesis" + + +def get_succeeded_pods(): + """Get list of all succeeded pods that are single user running pod""" + all_succeeded_pods = [] + + api_response = v1.list_namespaced_pod(NAMESPACE) + for pod in api_response.items: + logger.debug("Pod %s is %s", pod.metadata.name, pod.status.phase) + if pod.status.phase == "Succeeded" and pod.metadata.name.startswith("jupyter-"): + all_succeeded_pods.append(pod) + + return all_succeeded_pods + + +def kill_pod(pod): + """Kill single pod""" + logger.info("Requesting delete of pod %s ...", pod.metadata.name) + try: + api_response = v1.delete_namespaced_pod(pod.metadata.name, NAMESPACE) + logger.info("Pod %s deleted.", api_response.metadata.name) + except client.exceptions.ApiException as exception: + logger.info( + "Fail to delete pod %s due %s", pod.metadata.name, exception + ) + + +def kill_succeeded_pods(): + """Kill succeeded pods""" + logger.info("Starting inspection of Kubernetes pod ...") + all_succeeded_pods = get_succeeded_pods() + for succeeded_pod in all_succeeded_pods: + kill_pod(succeeded_pod) + logger.info("%s pods deleted.", len(all_succeeded_pods)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="Open Research Computing v2 Kill Succeeded Pods Cron Job", + description="Cron job to kill Kubernetes pods in Succeeded status that are very old", + ) + parser.add_argument( + "-c", + "--kube-config", + type=str, + default="~/.kube/config", + help="Location of Kubernetes configuration file", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Display log information" + ) + parser.add_argument( + "-vv", "--debug", action="store_true", help="Display debug information" + ) + args = parser.parse_args() + if args.verbose: + logger.setLevel(logging.INFO) + if args.debug: + logger.setLevel(logging.DEBUG) + + config.load_kube_config(config_file=args.kube_config) + + v1 = client.CoreV1Api() + + kill_succeeded_pods() diff --git a/ansible/roles/k8s-control-panel/files/etc/systemd/system/orc2-fix-dind-bot.service b/ansible/roles/k8s-control-panel/files/etc/systemd/system/orc2-fix-dind-bot.service new file mode 100644 index 000000000..b3f183dd2 --- /dev/null +++ b/ansible/roles/k8s-control-panel/files/etc/systemd/system/orc2-fix-dind-bot.service @@ -0,0 +1,14 @@ +[Unit] +Description=Bot service to restart ORC2 Docker-in-Docker when is not working +After=kubelet.service +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=1 +User=ansible +{% for host in hostvars %} +Environment="PASSWORD_{{ hostvars[host]['ansible_host'] | replace(".", "_") }}={{ hostvars[host]['ansible_become_pass'] }}" +{% endfor %} +ExecStart=/usr/bin/python3 /usr/bin/orc2-fix-dind-bot.py --verbose diff --git a/ansible/roles/k8s-control-panel/files/etc/systemd/system/orc2-fix-jupyterhub-bot.service b/ansible/roles/k8s-control-panel/files/etc/systemd/system/orc2-fix-jupyterhub-bot.service new file mode 100644 index 000000000..19c08eb99 --- /dev/null +++ b/ansible/roles/k8s-control-panel/files/etc/systemd/system/orc2-fix-jupyterhub-bot.service @@ -0,0 +1,11 @@ +[Unit] +Description=Bot service to restart ORC2 JupyterHub when API is not working +After=kubelet.service +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=1 +User=ansible +ExecStart=/usr/bin/python3 /usr/bin/orc2-fix-jupyterhub-bot.py --verbose diff --git a/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-dind-bot.py b/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-dind-bot.py new file mode 100644 index 000000000..daf580ae7 --- /dev/null +++ b/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-dind-bot.py @@ -0,0 +1,146 @@ +"""Script to identify when Docker-in-Docker stop working.""" + +import argparse +import datetime +import logging +import os + +from kubernetes import client, config, watch + +from invoke import Responder +from fabric import Connection + +logging.basicConfig( + format="%(asctime)s %(levelname)-8s | %(message)s", datefmt="%Y-%m-%d %H:%M:%S" +) +logger = logging.getLogger("orc2-fix-dind-bot") +logger.setLevel(logging.WARNING) + +NAMESPACE = "gesis" + + +def remove_docker_socket(host_IP): + """Remove Docker socket""" + ssh_password = os.getenv(f"PASSWORD_{host_IP.replace('.', '_')}") + + logger.info("Connecting to %s ...", host_IP) + c = Connection(host_IP, user="ansible", connect_kwargs={"password": ssh_password}) + logger.info("Connected!", host_IP) + + logger.info("Removing Docker socket ...") + sudopass = Responder( + pattern=r"\[sudo\] password for .*:", + response=f"{ssh_password}\n", + ) + c.run("sudo rm -rf /var/run/dind/docker.sock/", pty=True, watchers=[sudopass]) + logger.info("Removed Docker socket.") + + +def remove_pods(): + """Remove Docker-in-Docker related pods""" + logger.debug("Starting search for pods ...") + api_response = v1.list_namespaced_pod(NAMESPACE) + for pod in api_response.items: + logger.debug("Pod %s is running on the cluster", pod.metadata.name) + if pod.metadata.name.startswith( + "binderhub-dind-" + ) or pod.metadata.name.startswith("binderhub-image-cleaner-"): + logger.info("Found pod %s", pod.metadata.name) + pod_to_delete_name = pod.metadata.name + logger.info("Requesting delete of pod %s ...", pod_to_delete_name) + try: + api_response = v1.delete_namespaced_pod(pod_to_delete_name, NAMESPACE) + logger.info("Pod %s deleted.", pod_to_delete_name) + except client.exceptions.ApiException as exception: + logger.info( + "Fail to delete pod %s due %s", pod_to_delete_name, exception + ) + logger.debug("Completed search for pods!") + + +def get_node_running_pod(pod_name): + """Get node host's IP address running pod""" + pod_status = v1.read_namespaced_pod(pod_name, namespace=NAMESPACE) + logger.debug(pod_status) + host_IP = pod_status.status.host_ip + logger.info("%s is running on %s", pod_name, host_IP) + return host_IP + + +def monitor_cluster(): + """Monitor pod""" + while True: + logger.info("Start monitoring ...") + + w = watch.Watch() + for event in w.stream(v1.list_namespaced_event, namespace=NAMESPACE): + pod_name = event["object"].involved_object.name + if pod_name.startswith("binderhub-dind-"): + if event["object"].type == "Warning": + logger.info("Found Warning event in %s", pod_name) + if event["object"].reason == "BackOff": + time_since_last_timestamp = ( + datetime.datetime.now(datetime.timezone.utc) + - event["object"].last_timestamp + ) + + if time_since_last_timestamp.seconds > 5: + logger.info( + "Skipping because event old (%d > 5 seconds).", + time_since_last_timestamp.seconds, + ) + else: + logger.info("Removing Docker-in-Docker socket and pods ...") + try: + node_IP_address = get_node_running_pod(pod_name) + remove_docker_socket(node_IP_address) + remove_pods() + except Exception as exception: + logger.info( + "Fail to delete pod %s due %s", pod_name, exception + ) + + elif event["object"].type == "Normal": + logger.debug( + "Found Normal event in %s ... skipping!", + event["object"].metadata.name, + ) + else: + logger.debug( + "Found %s event in %s ... ignoring!", + event["object"].type, + ["object"].metadata.name, + ) + + logger.info("Stop monitoring!") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="Open Research Computing v2 Fix JupyterHub Bot", + description="Monitoring Kubernetes cluster to restart JupyterHub", + ) + parser.add_argument( + "-c", + "--kube-config", + type=str, + default="~/.kube/config", + help="Location of Kubernetes configuration file", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Display log information" + ) + parser.add_argument( + "-vv", "--debug", action="store_true", help="Display debug information" + ) + args = parser.parse_args() + if args.verbose: + logger.setLevel(logging.INFO) + if args.debug: + logger.setLevel(logging.DEBUG) + + config.load_kube_config(config_file=args.kube_config) + + v1 = client.CoreV1Api() + + monitor_cluster() diff --git a/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-jupyterhub-bot.py b/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-jupyterhub-bot.py new file mode 100644 index 000000000..97c9e22fd --- /dev/null +++ b/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-jupyterhub-bot.py @@ -0,0 +1,111 @@ +"""Script to identify when JupyterHub stop working.""" + +import argparse +import datetime +import logging + +from kubernetes import client, config, watch + +logging.basicConfig( + format="%(asctime)s %(levelname)-8s | %(message)s", datefmt="%Y-%m-%d %H:%M:%S" +) +logger = logging.getLogger("orc2-fix-jupyterhub-bot") +logger.setLevel(logging.WARNING) + +NAMESPACE = "gesis" +RESTART_WAITING_TIME = 120 # seconds + + +def get_binder_pod(): + """Get name of pod running Binder.""" + logger.debug("Starting search for BinderHub pod ...") + api_response = v1.list_namespaced_pod(NAMESPACE) + for pod in api_response.items: + logger.debug("Pod %s is running on the cluster", pod.metadata.name) + if pod.metadata.name.startswith("binder-"): + logger.info("Found BinderHub pod: %s", pod.metadata.name) + binder_pod_name = pod.metadata.name + break + + logger.debug("Search for BinderHub pod stop.") + return binder_pod_name + + +def kill_jupyterhub_pod(): + """Kill all JupyterHub pods""" + logger.debug("Starting search for JupyterHub pod ...") + api_response = v1.list_namespaced_pod(NAMESPACE) + for pod in api_response.items: + logger.debug("Pod %s is running on the cluster", pod.metadata.name) + if pod.metadata.name.startswith("hub-"): + logger.info("Found JupyterHub pod: %s", pod.metadata.name) + logger.info("Requesting delete of pod %s ...", pod.metadata.name) + try: + api_response = v1.delete_namespaced_pod(pod.metadata.name, NAMESPACE) + logger.info("Pod %s deleted.", api_response.metadata.name) + except client.exceptions.ApiException as exception: + logger.info( + "Fail to delete pod %s due %s", pod.metadata.name, exception + ) + logger.debug("Search for JupyterHub pod stop.") + + +def monitor_pod(): + """Monitor pod""" + while True: + pod_name = get_binder_pod() + logger.info("Monitoring %s", pod_name) + + last_jupyterhub_restart = datetime.datetime.now(datetime.timezone.utc) + + w = watch.Watch() + for line in w.stream( + v1.read_namespaced_pod_log, name=pod_name, namespace=NAMESPACE, tail_lines=0 + ): + if line.find("Error accessing Hub API") > -1: + logger.debug(line) + + now = datetime.datetime.now(datetime.timezone.utc) + time_difference = now - last_jupyterhub_restart + if time_difference.seconds > RESTART_WAITING_TIME: + logger.info("Restarting JupyterHub ...") + kill_jupyterhub_pod() + last_jupyterhub_restart = now + else: + logger.info( + "Waiting %s seconds for JupyterHub to restart.", + RESTART_WAITING_TIME, + ) + + logger.info("Stop monitoring %s", pod_name) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="Open Research Computing v2 Fix JupyterHub Bot", + description="Monitoring Kubernetes cluster to restart JupyterHub", + ) + parser.add_argument( + "-c", + "--kube-config", + type=str, + default="~/.kube/config", + help="Location of Kubernetes configuration file", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Display log information" + ) + parser.add_argument( + "-vv", "--debug", action="store_true", help="Display debug information" + ) + args = parser.parse_args() + if args.verbose: + logger.setLevel(logging.INFO) + if args.debug: + logger.setLevel(logging.DEBUG) + + config.load_kube_config(config_file=args.kube_config) + + v1 = client.CoreV1Api() + + monitor_pod() diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index b1ffc2321..2a707193f 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -86,3 +86,90 @@ kubernetes.core.k8s: state: present definition: "{{ lookup('file', '{{ role_path }}/files/calico/custom-resources.yaml') | from_yaml_all }}" +- name: Add GitLab Helm repository + kubernetes.core.helm_repository: + name: gitlab + repo_url: https://charts.gitlab.io +- name: Deploy GitLab agent + kubernetes.core.helm: + name: gitlab-agent + chart_ref: gitlab/gitlab-agent + release_namespace: gitlab-agent + dependency_update: true + create_namespace: true + set_values: + - value: 'config.token={{ GITLAB_K8S_TOKEN }}' + - value: config.kasAddress=wss://git.gesis.org/-/kubernetes-agent/ +- name: Copy orc2-fix-jupyterhub-bot Python script + ansible.builtin.copy: + src: files/usr/bin/orc2-fix-jupyterhub-bot.py + dest: /usr/bin/orc2-fix-jupyterhub-bot.py + owner: root + group: root + mode: u=rwx,g=rwx,o=r +- name: Copy orc2-fix-jupyterhub-bot Systemd Unit script + ansible.builtin.copy: + src: files/etc/systemd/system/orc2-fix-jupyterhub-bot.service + dest: /etc/systemd/system/orc2-fix-jupyterhub-bot.service + owner: root + group: root + mode: u=rwx,g=rwx,o=r +- name: Enable service orc2-fix-jupyterhub-bot + ansible.builtin.systemd: + name: orc2-fix-jupyterhub-bot + daemon_reload: true + enabled: true + masked: false + state: restarted +- name: Copy orc2-fix-dind-bot Python script + ansible.builtin.copy: + src: files/usr/bin/orc2-fix-dind-bot.py + dest: /usr/bin/orc2-fix-dind-bot.py + owner: root + group: root + mode: u=rwx,g=rwx,o=rx +- name: Copy orc2-fix-dind-bot Systemd Unit script + ansible.builtin.template: + src: files/etc/systemd/system/orc2-fix-dind-bot.service + dest: /etc/systemd/system/orc2-fix-dind-bot.service + owner: root + group: root + mode: u=rwx,g=rwx,o=r +- name: Enable service orc2-fix-dind-bot + ansible.builtin.systemd: + name: orc2-fix-dind-bot + daemon_reload: true + enabled: true + masked: false + state: restarted +- name: Create directory + ansible.builtin.file: + state: directory + path: /home/ansible/bin + owner: ansible + group: ansible + mode: u=rwx,g=rwx,o=rx +- name: Copy kill-succeeded-pods.py + ansible.builtin.copy: + src: files/cron/kill-succeeded-pods.py + dest: /home/ansible/bin/kill-succeeded-pods.py + owner: ansible + group: ansible + mode: u=rwx,g=rwx,o=r +- name: Add cron job to remove succeeded pods + ansible.builtin.cron: + name: "remove succeeded" + job: "python3 /home/ansible/bin/kill-succeeded-pods.py --verbose >> /home/ansible/kill-succeeded-pods.log 2>&1" + minute: "*/5" +- name: Copy kill-after-timeout-pods.py + ansible.builtin.copy: + src: files/cron/kill-after-timeout-pods.py + dest: /home/ansible/bin/kill-after-timeout-pods.py + owner: ansible + group: ansible + mode: u=rwx,g=rwx,o=r +- name: Add cron job to remove timed out pods + ansible.builtin.cron: + name: "remove timeout" + job: "python3 /home/ansible/bin/kill-after-timeout-pods.py --verbose >> /home/ansible/kill-after-timeout-pods.log 2>&1" + minute: "*/5" From 567afe4ccfc1ec5d09e1751cc05941a16db41bfe Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 14:01:36 +0200 Subject: [PATCH 031/128] Configure Helm deploy --- .gitlab-ci.yml | 64 ++++++++++++++++++++++++++++++++++++++ config/gesis-stage.yaml | 68 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 config/gesis-stage.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 70ce2d518..598e3281e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,12 @@ stages: - deploy-prod-nginx - deploy-prod-helm +.manual-web: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' + when: manual + allow_failure: true + include: - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.6 inputs: @@ -21,6 +27,64 @@ include: ssh-user: ansible ssh-key-type: ed25519 +lint helm configuration: + stage: build + rules: + - !reference [.manual-web, rules] + - if: (($CI_PIPELINE_SOURCE != "schedule") && $CI_COMMIT_BRANCH == 'main') || $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + - helm/**/* + image: + name: docker-private-snapshots.gesis.intra/gesis/ilcm/orc2/k8s:latest + entrypoint: [""] + script: + - helm version + - helm dependency update ./helm/charts/gesis + - helm lint ./helm/charts/gesis + - helm template gesis ./helm/charts/gesis > gesis.helm.chart.yaml + +.gesis helm deploy: + image: + name: docker-private-snapshots.gesis.intra/gesis/ilcm/orc2/k8s:latest + entrypoint: [""] + variables: + HELM_ENVIRONMENT: template + script: + - cat $git_crypt_secret_key | base64 -d > git_crypt_secret_key + - git-crypt unlock git_crypt_secret_key + - kubectl config use-context ${CI_PROJECT_PATH}:${HELM_ENVIRONMENT} + - helm version + - | + for d in ./mybinder*/; do + helm dependency update "$d" + done + - | + helm upgrade \ + binderhub ./mybinder \ + --cleanup-on-fail \ + --create-namespace \ + --history-max 3 \ + --install \ + --namespace=gesis \ + --render-subchart-notes \ + --values ./config/gesis-${HELM_ENVIRONMENT}.yaml \ + --values ./secrets/config/common/common.yaml \ + --values ./secrets/config/common/cryptnono.yaml \ + --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml + +gesis helm stage deploy: + resource_group: stage + stage: deploy-stage-helm + rules: + - !reference [.manual-web, rules] + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - helm/**/* + variables: + HELM_ENVIRONMENT: stage + extends: + - .gesis helm deploy + smoke test to stage cluster: stage: test-stage script: diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml new file mode 100644 index 000000000..6166f5f2d --- /dev/null +++ b/config/gesis-stage.yaml @@ -0,0 +1,68 @@ +url: https://notebooks-test.gesis.org/binder/ + +binderhub: + config: + BinderHub: + hub_url: https://notebooks-test.gesis.org/binder/jupyter/ + + LaunchQuota: + total_quota: 30 + + extraConfig: + 02-badge-base-url: | + c.BinderHub.badge_base_url = "https://mybinder.org/" + 01-template-variables: | + template_vars = { + "gesis_notebooks_https": 'https://notebooks-test.gesis.org/', + 'production': False, + } + template_vars['gesis_notebooks_static'] = template_vars['gesis_notebooks_https'] + "static/" + template_vars['gesis_web_frontend_framework'] = template_vars['gesis_notebooks_static'] + "gesis-web-frontend-framework/" + template_vars['binder_static'] = template_vars['gesis_notebooks_https'] + "binder/static/" + c.BinderHub.template_variables.update(template_vars) + + imageCleaner: + enabled: true + imageGCThresholdHigh: 80e9 + imageGCThresholdLow: 50e9 + imageGCThresholdType: absolute + +prometheus: + enabled: true + server: + service: + type: NodePort + nodePort: 30073 + servicePort: 80 + livenessProbeInitialDelay: 800 + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "1" + memory: 1Gi + persistentVolume: + size: 10Gi + storageClass: standard + retention: 30d + ingress: + hosts: + - notebooks-test.gesis.org/prometheus + +grafana: + enabled: true + resources: + requests: + cpu: "0" + memory: 128Mi + limits: + cpu: "0.25" + memory: 128Mi + ingress: + path: /grafana + hosts: + - notebooks-test.gesis.org + +cryptnono: + enabled: true From 0dc88983463c0d774e959f3006818ec3ce508364 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 14:03:11 +0200 Subject: [PATCH 032/128] Remove GitLab CI conditions --- .gitlab-ci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 598e3281e..bb2ca299e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,12 +6,6 @@ stages: - deploy-prod-nginx - deploy-prod-helm -.manual-web: - rules: - - if: $CI_PIPELINE_SOURCE == 'web' - when: manual - allow_failure: true - include: - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.6 inputs: @@ -29,11 +23,6 @@ include: lint helm configuration: stage: build - rules: - - !reference [.manual-web, rules] - - if: (($CI_PIPELINE_SOURCE != "schedule") && $CI_COMMIT_BRANCH == 'main') || $CI_PIPELINE_SOURCE == 'merge_request_event' - changes: - - helm/**/* image: name: docker-private-snapshots.gesis.intra/gesis/ilcm/orc2/k8s:latest entrypoint: [""] @@ -75,11 +64,6 @@ lint helm configuration: gesis helm stage deploy: resource_group: stage stage: deploy-stage-helm - rules: - - !reference [.manual-web, rules] - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - helm/**/* variables: HELM_ENVIRONMENT: stage extends: From 8290bba70fc40b22a31e67d5705c210a2d6ab053 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 14:05:18 +0200 Subject: [PATCH 033/128] Fix Helm lint --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb2ca299e..1dcb756fe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,9 +28,9 @@ lint helm configuration: entrypoint: [""] script: - helm version - - helm dependency update ./helm/charts/gesis - - helm lint ./helm/charts/gesis - - helm template gesis ./helm/charts/gesis > gesis.helm.chart.yaml + - helm dependency update ./mybinder + - helm lint ./helm/charts/gesismybinder + - helm template gesis ./mybinder > mybinder.chart.yaml .gesis helm deploy: image: From 5bcf27d0dec308f9a5b80c7b83bd6de27d7c5f30 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 14:08:12 +0200 Subject: [PATCH 034/128] Fix Helm lint --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1dcb756fe..c63d79099 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ lint helm configuration: script: - helm version - helm dependency update ./mybinder - - helm lint ./helm/charts/gesismybinder + - helm lint ./mybinder - helm template gesis ./mybinder > mybinder.chart.yaml .gesis helm deploy: From 9d1201ab4c4d391601708db450086d2aa260ade4 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 14:15:11 +0200 Subject: [PATCH 035/128] Move helm lint to deploy --- .gitlab-ci.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c63d79099..cdc3466b0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,17 +21,6 @@ include: ssh-user: ansible ssh-key-type: ed25519 -lint helm configuration: - stage: build - image: - name: docker-private-snapshots.gesis.intra/gesis/ilcm/orc2/k8s:latest - entrypoint: [""] - script: - - helm version - - helm dependency update ./mybinder - - helm lint ./mybinder - - helm template gesis ./mybinder > mybinder.chart.yaml - .gesis helm deploy: image: name: docker-private-snapshots.gesis.intra/gesis/ilcm/orc2/k8s:latest @@ -47,6 +36,9 @@ lint helm configuration: for d in ./mybinder*/; do helm dependency update "$d" done + - | + helm lint ./mybinder \ + --values ./config/gesis-${HELM_ENVIRONMENT}.yaml - | helm upgrade \ binderhub ./mybinder \ From b2f402253dae3e6fe653dad974b3b9558bd9167e Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 14:24:27 +0200 Subject: [PATCH 036/128] Deploy secondary Helm chart --- .gitlab-ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cdc3466b0..0ef94245d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,6 +36,15 @@ include: for d in ./mybinder*/; do helm dependency update "$d" done + - | + for chart in mybinder-kube-system mybinder-tigera-operator; do + helm upgrade \ + ${chart:0:9} ./${chart} \ + --cleanup-on-fail \ + --create-namespace \ + --history-max 3 \ + --install \ + --namespace=${chart} - | helm lint ./mybinder \ --values ./config/gesis-${HELM_ENVIRONMENT}.yaml From cb94204cdb22fce671e376e504f1eef594d2adca Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 15:01:55 +0200 Subject: [PATCH 037/128] More Helm configuration --- secrets/config/gesis-stage.yaml | Bin 0 -> 10989 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 secrets/config/gesis-stage.yaml diff --git a/secrets/config/gesis-stage.yaml b/secrets/config/gesis-stage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bad5770e2a7d7760954407ff1a56e60786d4df04 GIT binary patch literal 10989 zcmVU{AqbyY2dUU`W1fpcBQ#REY$paRa3Y0D>ZZdS8rDZ`Q zvT7hTbo<+ZRt$qa@I^W?21LL%&9aia*j?L45u3>Kn6R*7sBAQWp!SuUq4xjl;l8L% zshD9b*W}laF6$&uuv*QJ?w?}@w&V39 zmO;)kDu(}kdrNS4K>+wAT8|hIOb`%kc&6(iiZGonowxq|EP{A(LT>nq~LP6a&(G_Db z#RR)al(C!Gl^Vwo?RqcdayDMQ496LaH_OqMGhi}roB2!cl>(^x`u`mo=u{>e`H+lFP6><-$z_w^?p*l;M z0G4VIa2noUVm$DdO6qDjclCs34pGMFrgxdi(di6W35PNi-AR91(i_ax0d{9e0u^%j zJiRKlj2lv>jlCv=>ph4*aGC1extA1hjWP$?&wBOs&85uGJ5itd@d!_WT{S0^5{k9tkOLQ)=7wE)_ylN}yY~%O{dT(m z!Fy}rfk30gG7&NCg;@OTV#-I)nV=Ke<3GfUAaQe8=>R(j#K#S~oj6O2z2H#$?m{X) z=)X+y5$)4%-q-*|%omyN%8?>38`M=uI%Bkq_Rl}16}7N6XqD_4!m?C}vxZTk@RLY6 zv;@g1Q(qRF#(%WYQqtMQZt<)`R{LtBDC4S>7|;A|5Dq%A_)~RDykAmTid~@8D(OAa z6GBHt=sjL3r{vdA4x*lkk)abNesDP)$*iK}(k|0NkAvOv+S z*TUF)+mv66eK`$QV}r+C=FX~ha#k1)=W)QP zg+M@QR^LhOXVNf$6vkGePF{gy3WBdpf$DTBj@D%R8B93*2P01FC zp5dhwshgVfV8SZ#><5BH&hA?>&OTyG=J!FBF8>RpNDgYCQ1wOimyKNix*2Y7Rb*Fn zFBs4GZ>CXiikFx(&p(lMq!!l9rf*verKWXkW?^Vt5+Vb;#DTF!JDXGJ z5-V8;xmy{xM)M(`8py0L|J03J>PV){BDy1HjSf;CxZD2>4c_o}w?}=b(OM zd4j<2`pW^Qg+?-*=C5=T-J4m-`tzvLjEB91O1;U?8OXu^AEh2&v`HpjVA0My;f04s ztmWFZB``(zZ9z&q78o@;%eQttU-qH_sZQyl@gsU-mkBY|qvVmjyKUvVZp=Kb>wt2P zBPq#(AEVk>nlcFGLEwCXHPwC#I*vUI3v-EcbOblCjcg(zG}!eKy-ekdZ`=ko1SjDDuOYPw5gGr#xwxf}>~Yf;VIu zy3Gz+XZ09?nXVV`P@m_^73^;GOWa5khz<|*wYL%fMI~b3qQ84;n^x8OqmjAhNl%zc z@{*-(%g|gI zJp4{`Iv*QH;QLv8BXOEB;RrL}AoPus#R&VRQxVRYsTr*F)nV~Qx|7>2A-+rsM0iQ0 z&0NC7zlFTRBkkWx&mE>f`RRPia0juoY&Ft$LpsOuir#yMw3yTcubnbquX|J8(GKc$ z5tH;2vJ9<1<6DjWG4&=2M_84c(srvRKPmj(HHYXQP3d48-G*GNj+R0YKkW}0oS;76 zc)=sd=w{B(b(sIEuy$pl@POFM(z;c`2VL`m-?}-Dg}oTzt))8kG1 za%Zr>K>ZJ7iu{_y1>g#2k1+-JeNE}I)JPf!8m%->HMhFtH3KIdkuk@`H9v?s+IEw^ z4aW(b?xRqgau2D?3Iit(IZ~N2M!(l}&a+sVhb(?mEv|98K10}<5FZC!@3}`)ZPS}x zamktiHsUcYo>Ha5KcG(IVUP5#74)1Q6Au=)Wq?)F|uHA|l02 zr0KQ-GfjEmE))xAK04wVV@Lj(nm6ng*KyCWFCRa3>$7zFZ_c+b*UwMCS^zeu#D^(A z?(`g%#`mKHd@?}wwR+3S4z@7{OqkR-=`^YQvl`L5Ps<^VBRUQ0VR)z7k+uo?pJr?-# zdOX24=GJs*^6#dX>@Pe*Y&Ud0ja=wZUL@(?*ERKesFEh%C zILhtBGzg_b$I{#L*Xw9i;>Y!sbjC#?YFz~ie95}&!6~R zjW44;@);cBx+HQcydQNK<)>%^Nt<@Y_M&jNCCxBMDPtKtZ*^A*T3z9a`Q#pi3QQCW z?O956U`F|`3irFmhPZYpO^uXy;qop_j7+X^)hN0;1e$ZW?xUlDn#wdo!MInw1<}Cg z{kcC6*h>EZzSRrA7cRzaZ1_6;Xh0~Vj0d|Tl z_wYo8<4ASODstleHZw~ntM*$Q;t0{_Tyl^V)I5#WW16zPpO#M$u`r=$;BOjn&+|jH z#Er}zU@6Q>bG0Q?7OFQ6SDclw-v708WJi)y!u;zLqF1L$CWbOb<7~9_`vKnvrB3LW z?q9^G42;KgKWwt=hREg~Zs50E&ItB1gir!FBxCDsY z*SIe5(nF8Asi;2SEECY~V#;nOcL{NFt{ve~b2y;?I#{b*+dt~H!BO)aV8})|38?PW z7-cFQYhORmD<`#MJe&RmhMgiuBgAuhb#wCdE%zz+T^nsJUA7B+$Tg@QxO^l9hY89H zTUE(9_o-nK6*@ta-tI@wU5eB$$U_vdv{mt+jlB{4+axREH$0oLVOru-jF&IZ7ZVyb zgmr;z9U@apI4&tk=H-J7P5i>D4=DU2aBxAC3bbDN0tI6}%(|LJ`Ir6(azT55ZH9;$ zDa<1u4L{zlX|~XGEW$v=jXg!A&duwHc(>o!hu{kvtH$G(>|rs_U|{5>*8%c&1kUtov#H~gzR+gOi<5-B{i`d!OZTC>0 zpWNf^STbB(zAv^~4Z$FkMu*a?6CB3j`(=f&=Cbl&;>!0mEU4Pq3o$;baMP~R0R$6t zwav#43T;6Vgxxl*G9kVknWA>y+H1tjfo1xs9=ppw53ic71If>kPGr_td64fg#9!4t z3gQCR8F$_KZeWR zdT`(*xUEfZFp4uB*lrVE5Ug21yq;IFKo+s9;%|V6NNPE09fb1U-;BRKU?%=jv1bcW zC1n=-apsSkc1G)!k?ef)USX#EpqqzsKp(0ek)1}f_S^S}91)Sjs93oAZ zwplhx6vPtNhCcinW%PQM&+T(PuYF8%ZJCRvFj9%y9oZfo z^9!$I7<%BLAjWUneDsi#=VOTcw1$ssSJ%AJNd#L~7(1Xm>#`j<*ZWUX9DW;CG+Bhd z9GPcxn%qu0mvfBpLQ=*|dGz6vpOTN-xMt}X+3S%r>0~!yR_IVjycvG%N^rx%Ck$AW z&expI07|TjUY=2VLUunfq(!E|@X$wJA*+CLnn@c}!nhweqZWpDg)|)ICeffzWN?o;zNpt(6kj-od?7-%8-;P z#JxHphy^bZbE^ZQi|vzs5wS~wT}6#QIIv#WUSN=ia`NTK-7pEYIOSs05$)$Mgb8{n z^^i*nNxBr^02ww;--rxP4#H+(?QeX1Ib0l);0lXsc*m|ZPpLqz37BCz91mchF-h8v zrMgSta2J^wwXbm}B0^yQo{B^stA-!aC?ukksc7U33-3Q&hnlTMh|;kM$IjoUSlYt4 zqwI2GcW?u2pLjHTQPTM5p|Kq+-0{GJj0E>gr={U4rK}Km=Rez z#UQ~6B^X}uCi~yJ)R2)d2egow6$k@4l`ptB)hU3fS3Y+r&*-6P@CciU=4ivFJf4j& zL7-*;q%p$h%28^uRKEr^9sFW!gOo;ypsj1cuzn6aG|1EUDVRRM~zl_UP)q+ZCv>?=%vXuL=c+}@AX7wLneh!JQep3A#j)ziH zYFcLP_JlZyaGNS@e#UzA+mzS^U|^Qpz#5E3Pk*F7Q?WncwFdoJtkht~unoB9D-f<@ zcE!Xh-x;UMJ@F68Kk(t=Md3Y$=2k11#%^-O6(KKJg2eOBB3yS@cPdSDX4{rJZUY~( zcy=g+Le7D-$AOj-#oB^HkhYGx^ib9Td6!0A@9k1D;GDR(h5(De4-nX@&5#ik8ivo; zFLt|B8_S~DDwrJq&nsw-!t4zrNik85<;`kE!MGt zS?P&iafcguU<8sa51aJ*g}oDcCpo0v>8<$GY1{wv&ejguRm(Gr3@1yDO;oOCPYHJW zE9}~5nDd{7Mfl>r?`kHGTm@I!))MUHrdbEQ+G7XH@TVSE4z%xvZ&ptJUo8cVBSd0^ z%mFDi)j_c*El8rpq2My-C~SZ2r*~?!3uH2Suq~Tfgjw?JYN2ZJi|=%$2k^l;AZ5gxeF9f(L!2}IthabNR-CSnGt zJf9D+XR$i!J>2xh+N5CyaKw5`j!mUsOa^-qPU5VRo<4m9j!W_w5^7Rv6 z?Id=;Y@4wc6vzsAS@@7{2md>&H5ICMskmc-=D$;3?Ry*Go-=4D2XC_e>lpe@CvW*DRZirKWGfiuovJv2$HvpB$`1c+ketFw1LpUma3Oeecc`Ygii+j1 zDwum}Z`I%8V=R2SBW162wkmeoR77msS$g6afZu+$Q(V;ldS5K+_RpIrcxYZa2eVP9 zU;6ZoDv}1B25Wk}!s^1qqImMsu-YeEu~OQ@+qCpW?^$+J6W z=6~z9t|@%|dOhB`NK|&_=~hdoRHE$01nf)rvC56S&eUPm1=fV_jE7>g3<5UH1F8L=`;^Eb@iC`ncHvTPdOS3Rx zd5sbalEh&uKKhw)_??fy5uj2$g*-~Jso`Nf@@&PDc##N*{|VJ|GC=O9J&%)_3y#v6 zKw{Wz!L^{zcg_0m$glfIvIZC$m?b?DA8%2Tfj*jNrj63f1rQtyp;?7sRC(%*dR29C&=^awFrXvhz9K2cwi_~{9t{S1YVX?Ix z(cBiUUazMiYQ?4?O>&FBpY_4jQUI`m9{t47Q%kyb?yJRik<>;Jr^FC^SIBeQeo6Yu z);>hTWGngPWE*TpViO5|q0JSC5d9$VmuEf-ACOwpO7h@|rm>$SQyBOPTRi6AE4U5S z;0_)kdF@1NsKd%~IC9Pn7b{FLOHvF@JJq1v#R`}Su+<0I*^sAfuh%4F=?pA>rmZ`L zkOqmLCc6$YqHS572|Qw{z9W%NM$`}$8yAH_eT3*aoh4ZHA)yq35j42DqZsMn`S*&t zmBnbwXyarIg18+wNLp#DyXqiBfc#0HS4L_zq^ZMxty_^oGGLMHwT+<4y;(W6EL`E= z!EFylhKE65^e#@G%;Ndo^WRma&6LNq-LTn>**Emo3!nTKT3ML*hAPiD2Wg0lV}5jq z+mC((J4J%>4Ie+hfZ&B-Dd4SU62t7{z3Hf{WgI|=>kW2Lb8yXy3!1=ZsT_Ah4|*`D zKOm(#+@X#`=DJ_sYU{($u}jdMWb)Ek6voFWhtH zO{oJ>@*~zk@g@Xm3Z}bzk$h}B$mhs8V{Qd~+dF3!@jpX*H_FaOnVBKtJ29agG`E{i z_qS6^rCuFMi#gL6Cg^gZ9|tgHXrd3~Lk|SDoZEaLSe}?dYWHyaI3p>J4|3*7L`}??w%Fvtr*{QQ-p*gpfzS^Zl7^(x_gULml$-tm{`p?t)Tup zya7zBP3tf|HtMJY66J04@zHV1$v&FG-EG50hHsb$3rboceLIt5B1t#UL)KdRUqHYt zIw+g@R5)s&0#J{=)cP+$a!FfGktg)tI{hKTa~2#U^J!U3i|GLL z+|lFKK2D5?jAaL*iCO~=d(l+r8$Dez%>4LIvEo=I+KzlfHr7VqeNtZ8o|VE=z_+tEol*473NmcU=`+>RGrzvpBqf9 zuwqD87jBCMTn<>1dgqKENZ|u6Z&=1}L}3~ux&+ZBIgW)aj#YioCpk+f+vjpewfvM8 zM>}7jT&&FcG*qz9az)^3CsDx}5xavuJVp^J5{mXw8lZLuuvV<_ZoQCov3`zcxot8> zbt6Opw+*UjZR@_0Vjr6VKr(P}nCmZ%?bGV?$^0#tHt_b`KEs-8pn4Ie_}d3WHuqKT z@>ZxK+XnC)H#iO(gThb@QLf{%u|FfVZbN@kOX#;|4mal@m>4|(O|5))dRDiI-q;bC zbbo(L{<_|?-wA+ZheK{J=I-IUagVA#316o83EfL$>Z1U`jdPzEz5r8c7!Yi>_E*UM zASuYsEeTwu#w{qS|L*WPR{Oc+lYaTOjtMF=1@mTXEbo0v-q`Y!`c+IxV$VnWg@Fx# zcE)7Y*o54NNN_`#N4Wg@2@NY!&!IXPF>0=EC?2<>A@KG3xDoS9{lYJJToxDIu zjQNN32VzGn=)-fg>9$nl>gKlWW}swmJpm_xD?g}smzK|XN{ORE_Q2JaS&r-4;~shj zR|D;LEs4&X_xlf2?0DbYZr$WBWTkc#v%<3|EOO+~oE zs>a*z+MA{1h!?3&MOEBU)h@_ajX1Y(ZS@b0vQiNJz+t&4nJG%BM8_+etSsOx7oz`~ zOhqoYI!IS>q4#@XvNySy#ff-O=o;~7P+(4d-DoPbF&ppO=^Q_?yw?0Se+rRU! zC7J1$j~jh)t*sxG(H*eL;$OG2ro&!^QYqp^(X3*X_mL;_%~sUF_zl$H&_uiIlquH? z2j7TAg~?9mx|7q!oj^zmXF@&(bg!p>2O&}i9e*oVB~GwJNA-O{kqapC#vldP!}jgB z15PqgE|sm@YEO3tSwou7M59z!yw81twnF`f7OQPZ$cY6NFawCD%hs_*fp8vHmxtYh z=ma#?u@2#Pq=8OTgqHvM)6i(G3waR&om5b=Nwg&TA}_3zA&KxVDMjmBrB4v5LMqG^ zUV%*21?6%@GKhVtd`phydag))TXMYLEBOLx0_H|$u%T1I1GIgiqoq`3@!3Msdu$85 z(4K?7=vKHVpe`2|M!Z zTDMt(g3SI*&4au(_ulnwfrri?Q;-bYp{M0uX&tRb;iBZYV@*X?n_nscjLK&<*dCX9 zai@RKGQY82YaN0^eqB&OLY;9;6Nrm=S|%dKuk(>m9w1TzRgj+t-ak!GOrAOVx`OE> z7I35)bBY!G;oxof4)09`FQ40WALy+C7Z0IJB>qG3o z=Gu{*2B56<5oYXNauWf>u@G(r5pt+U^_+tu98cly2opViT$}pSiFDZnMSOeEtVdsP zGxINI$B8>ppBmPEpJ|c%lms!3%Z4C6M1(C#gD8K5n_%LlvKV- zdP&}X3C!7?3+aLFBFlVARvCH+W$H!|Ag0!%K34#!F@BL;hKT$bGfdny1#sK&Iqu%R zYS|6Ca6K($O+de3e9`8h5+s-ulFi`)Ax7F8rfk2`(Uy*WxL}2I|B6Z{jq9@RHw`L6 zv4*{}F?>;RD&COl2Z|8EL}T%zeLHE{)4Xl!m_Du*KChXnw(5uiFl}*)xFN#R>gw!$%}$~rD`32BK*GXai#ezTT*}3Ov2w+ka6?AtMxyq+&jSlc ziE@&Bi5LbQ+dbHWPP|0VFDppd1#;lx5t7btB9;qgrbLot!x0|H7LKoAcTKfLW})Z} ztuX2w+A`cQN}@a6vhI37-Fm3Gl%y95k>GbaixGiH4yB?1aCu#(N$Eee_V?)y%VSd& z8D{bUbY6Qh+GUs%v{lpSqO?P;O|F2CL6Kl|>%9#%`+`sq)52%JkbJ2fs9hHtC!e?( z#A`&DbD6k{_p>&L#g}OY#!+vXwpEw1!;V7K4-UHPFQKPb#Q0GIE1avHDyYFsAO{Ea zxlgosG?wkr$~_yR8Hnm?;T`!)2H}x-aNw`?UTmiC3YL%SLR@HLbG}ZRKPM@~C01Lx zyZ@RE@OCs-;%CdXWKLw${LS~7f@Oz#AkyjxBr;rrgYgh8x(U&xGFb~^6;@tkn{(~h zjwx#9b^&nk?{v0Du?Br#w%CbA@kYwY^tzdNc?wT8NULK`zw@l0y%OP9gd zta1aOlnq9zeV?}^wnsHz>`D$VxX+&8IvO@qw$hX00U!eY^8xqbtQi%r87Pw*SaEJo zG!k&@I|sWpPhpU+3PfeCTS1{h^N$0A5Fsa)Tiul^Uu4u-7Floc+UqHeg(!(eBgBe^a8aPQ5M%eF8@PjL}!6*F2z~Q-Iw%wRYLkVR{SPTWU=kE#*BQ zjf=W4Li_c{EQN`~&l%2-icPgUgLQwAvSlfZb?+>o*44oDXPUBiaXvM+6fW}=4jog7 z4#aE&1e+{G*X1SbL*tZjh5b~F%?(ULH|9(@wmfZobbx$O9=3N#(nxGLyF)|(Oo!W& z-NyTp0jAp6tXVErv%;K)`HbV3+->nsQo^E5JaKfa@=FWl-w4U@#JDUxrKJYTwoYz{ zPN~Lzc7gs4A4+^H_gvZPbkBN|OERKuozY{|(bf&Lz{7ea^URaqK7(LM?lbblX*(>I zqN)c8xGpZU(h`liZQhvyZ}`0RCRz(OWdw)EVKVMkq4FK!Fv$zI-U z_mHkr>}N;yE8yI6<`bRZBW!od1M&6~b!EO4S9P&TgErr-=r-RJqzBM;yo$?o?2aV!gQg&HGcp=EE1=wEe|HZ_`sVjevu&WXC!P!sW!dSgoIQ3wYtUB^>eTa|9@2fzpWm>7a$&?dMxZV{+7*Yu<oNs!c7Lx&Z-5R%3L z89H(fE4bHx}hlsl#Hv~f>->swxNHyZZ}p#&tGl~y$fFmWvOAv9&a zlE9N7NB*#vE8G+fOOlhnf#1*W6_jPSyYd1kShGaNq6@( z3PWk5KQHf{Tzzzn2Qr7qp2c^A(2inv9~cF+0evr#C^8ud4n__RPN4jh;yD|W5so~~ zYewpA*Y{)fejigN$dtdJRzkeOOsiAeR=W+DMi#%Sm5?z(RQ&vzfs6N(knogma8 z0Vp`d#q#!xN7#>wxsC%`e$btq0m+}Sg=yqtnI!)PP6^v&`Hg0#k&xz?sOmRrg4(+9 z`my56+Sz8tJHEw(1ME4&2{l#;uXpe58wYOEr%mGDo6opgqGAuEw~=BC+f93KUS|L) z0iFbWWjV&(CS;1c^yc_4sHsr+3sL|xOUv6oAGAG6j9i*zq#>}k~7Y0qTF3l zvy~5Ab)}7v=(V-%dBi}SL{jU4?>11D3Iq@tnLN?4HJ9U6JRP`t6Ch?m()Xo@)k@y} zfv@-*Oxf<=TU!1`%m__eZhdMr*xx`2Yu%9Dr8|PNO%FO84OgSB>lJvTY`KNp`KQrO*_&y+_gH6$mDeZMF_S0A;? zYE(zJZZX3l_2f}0MKiYPHbMN{H`?{&uyd=Cs%_^{dKK)dN9I8#cmBJizwn|Ss-=6d zo&WXs7?X*@!2B7mU}M4ejKcnoP55Eqoy{P)?FUMUQ@UwhbCVNZn-4i`&s9eJ-O%7E boJ1`da_23!Xo@2?JpPq~x7bP9Qi2LXR0c+( literal 0 HcmV?d00001 From c89ff4bf71a2d6f1da52a40bac5a85c1dbcbbf64 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 15:05:33 +0200 Subject: [PATCH 038/128] Fix Docker registry --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ef94245d..abb0451c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,7 @@ include: .gesis helm deploy: image: - name: docker-private-snapshots.gesis.intra/gesis/ilcm/orc2/k8s:latest + name: docker-private.gesis.intra/gesis/ilcm/orc2/k8s:latest entrypoint: [""] variables: HELM_ENVIRONMENT: template From 2e491b633a536427fd2d91b7cc49d5a50507a451 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 15:07:47 +0200 Subject: [PATCH 039/128] Fix shell for loop --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index abb0451c2..b3b2fdc80 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,7 +34,7 @@ include: - helm version - | for d in ./mybinder*/; do - helm dependency update "$d" + helm dependency update "$d"; done - | for chart in mybinder-kube-system mybinder-tigera-operator; do From 4390d7fcddd71436ed87e0bdd5ecc29a340acb2b Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 15:10:44 +0200 Subject: [PATCH 040/128] Fix another shell for loop --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3b2fdc80..417ebc545 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,8 @@ include: --create-namespace \ --history-max 3 \ --install \ - --namespace=${chart} + --namespace=${chart}; + done - | helm lint ./mybinder \ --values ./config/gesis-${HELM_ENVIRONMENT}.yaml From 2abfa0ab95038b1ce040eb48785b46be0609eb33 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 15:16:47 +0200 Subject: [PATCH 041/128] Fix name of helm release --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 417ebc545..bf1e5411b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,7 +39,7 @@ include: - | for chart in mybinder-kube-system mybinder-tigera-operator; do helm upgrade \ - ${chart:0:9} ./${chart} \ + ${chart:9} ./${chart} \ --cleanup-on-fail \ --create-namespace \ --history-max 3 \ From 3c9aaf1eb5d9051424cd7c3b8ecf2417976e4e8c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 15:31:10 +0200 Subject: [PATCH 042/128] Add hosts to Helm configuration --- config/gesis-stage.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index 6166f5f2d..e0d4f2ba9 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -1,6 +1,10 @@ url: https://notebooks-test.gesis.org/binder/ binderhub: + ingress: + hosts: + - notebooks-test.gesis.org + config: BinderHub: hub_url: https://notebooks-test.gesis.org/binder/jupyter/ From 165d0602f6f0a1243938604251df7dccd8550971 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 15:41:22 +0200 Subject: [PATCH 043/128] Clean git repository Related to https://git.gesis.org/methods-hub/interactive-environment/-/issues/6 --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf1e5411b..b960ec1d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,6 @@ +variables: + GIT_CLEAN_FLAGS: "-ffdx" + stages: - build - deploy-stage-ansible From 44556ab24df0b9fe0d8286303b9d4b5362537bdc Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 15:44:16 +0200 Subject: [PATCH 044/128] Do a clean clone of repository Related to https://git.gesis.org/methods-hub/interactive-environment/-/issues/6 --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b960ec1d5..1702c3a2e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,5 @@ variables: + GIT_STRATEGY: clone GIT_CLEAN_FLAGS: "-ffdx" stages: From 3ed6fb9fa2920958183cf0ffe05dd5a6e0bfe648 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 15:54:29 +0200 Subject: [PATCH 045/128] Fix Helm lint --- mybinder/templates/minesweeper/configmap.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mybinder/templates/minesweeper/configmap.yaml b/mybinder/templates/minesweeper/configmap.yaml index a083cb0b7..41fd2a95f 100644 --- a/mybinder/templates/minesweeper/configmap.yaml +++ b/mybinder/templates/minesweeper/configmap.yaml @@ -1,4 +1,6 @@ -{{- /* configmap for minesweeper source files */ -}} +{{- /* +configmap for minesweeper source files +*/}} kind: ConfigMap apiVersion: v1 metadata: @@ -12,7 +14,9 @@ data: {{- (.Files.Glob "files/minesweeper/*").AsConfig | nindent 2 }} {{- (.Files.Glob "files/minesweeper/secrets/*").AsConfig | nindent 2 }} --- -{{- /* configmap for minesweeper configuration from values */ -}} +{{- /* +configmap for minesweeper configuration from values +*/}} kind: ConfigMap apiVersion: v1 metadata: From 34caf68cfc357efaa05c44fc3d91b9a42c8550c0 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 16:03:47 +0200 Subject: [PATCH 046/128] Install Cert Manager --- ansible/roles/k8s-control-panel/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 2a707193f..c4f30f0c4 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -86,6 +86,10 @@ kubernetes.core.k8s: state: present definition: "{{ lookup('file', '{{ role_path }}/files/calico/custom-resources.yaml') | from_yaml_all }}" +- name: Install Cert Manager + kubernetes.core.k8s: + state: present + src: https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.crds.yaml - name: Add GitLab Helm repository kubernetes.core.helm_repository: name: gitlab From a4fd80503ea1fe73ae5bfc53c64f87cc8fc3ff29 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 16:31:22 +0200 Subject: [PATCH 047/128] Add ingress-nginx --- config/gesis-stage.yaml | 14 +++++++++ mybinder/templates/ingress-nginx/ingress.yaml | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 mybinder/templates/ingress-nginx/ingress.yaml diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index e0d4f2ba9..695497131 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -70,3 +70,17 @@ grafana: cryptnono: enabled: true + +ingress-nginx: + controller: + scope: + enabled: true + service: + type: ClusterIP + +static: + ingress: + hosts: + - notebooks-test.gesis.org + tls: + secretName: kubelego-tls-static diff --git a/mybinder/templates/ingress-nginx/ingress.yaml b/mybinder/templates/ingress-nginx/ingress.yaml new file mode 100644 index 000000000..8553d33ba --- /dev/null +++ b/mybinder/templates/ingress-nginx/ingress.yaml @@ -0,0 +1,29 @@ +{{- if .Values.ingress-nginx.service.type == "ClusterIP" }} +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: {{ .Release.Name }} +spec: + controller: k8s.io/ingress-nginx +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name }} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + ingressClassName: {{ .Release.Name }} + rules: + {{- range .Values.binderhub.ingress.hosts }} + - host: {{ . }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: binder + port: + number: 80 +{{- end }} \ No newline at end of file From 25b3e6191ab625c7067b25b23af65ca9f21d67cd Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 16:45:21 +0200 Subject: [PATCH 048/128] Fix NGINX ingress resources --- mybinder/templates/ingress-nginx/ingress.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mybinder/templates/ingress-nginx/ingress.yaml b/mybinder/templates/ingress-nginx/ingress.yaml index 8553d33ba..67c691bcf 100644 --- a/mybinder/templates/ingress-nginx/ingress.yaml +++ b/mybinder/templates/ingress-nginx/ingress.yaml @@ -1,4 +1,5 @@ -{{- if .Values.ingress-nginx.service.type == "ClusterIP" }} +{{- $ingressType := index .Values "ingress-nginx" "controller" "service" "type" }} +{{- if eq $ingressType "ClusterIP" }} apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: @@ -26,4 +27,5 @@ spec: name: binder port: number: 80 + {{- end }} {{- end }} \ No newline at end of file From 6c20fe1fa02d1ed75ebaeee88ff075957e36bd2b Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 16:59:21 +0200 Subject: [PATCH 049/128] Remove duplicated resource --- mybinder/templates/ingress-nginx/ingress.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mybinder/templates/ingress-nginx/ingress.yaml b/mybinder/templates/ingress-nginx/ingress.yaml index 67c691bcf..fbdac38e7 100644 --- a/mybinder/templates/ingress-nginx/ingress.yaml +++ b/mybinder/templates/ingress-nginx/ingress.yaml @@ -1,13 +1,6 @@ {{- $ingressType := index .Values "ingress-nginx" "controller" "service" "type" }} {{- if eq $ingressType "ClusterIP" }} apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - name: {{ .Release.Name }} -spec: - controller: k8s.io/ingress-nginx ---- -apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ .Release.Name }} From 824fdac459868f8d3b7f3db8ecf064b145d0fa31 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 16:59:41 +0200 Subject: [PATCH 050/128] Change externalTrafficPolicy Service "binderhub-ingress-nginx-controller" is invalid: spec.externalTrafficPolicy: Invalid value: "Local": may only be set when `type` is 'NodePort' or 'LoadBalancer --- config/gesis-stage.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index 695497131..e78054fa6 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -77,6 +77,8 @@ ingress-nginx: enabled: true service: type: ClusterIP + externalTrafficPolicy: Cluster + static: ingress: From fed5547c64e89075f60a153e77bac09e8ccc07e6 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 17:08:55 +0200 Subject: [PATCH 051/128] Reduce replica --- config/gesis-stage.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index e78054fa6..3d84e69ca 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -1,6 +1,8 @@ url: https://notebooks-test.gesis.org/binder/ binderhub: + replicas: 1 + ingress: hosts: - notebooks-test.gesis.org From 1b006a5ed3a95a34fdae889598a6f61c1335d71f Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 17:31:51 +0200 Subject: [PATCH 052/128] Remove externalTrafficPolicy --- config/gesis-stage.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index 3d84e69ca..24f0da107 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -2,7 +2,7 @@ url: https://notebooks-test.gesis.org/binder/ binderhub: replicas: 1 - + ingress: hosts: - notebooks-test.gesis.org @@ -79,7 +79,7 @@ ingress-nginx: enabled: true service: type: ClusterIP - externalTrafficPolicy: Cluster + externalTrafficPolicy: null static: From 0cd6d5855dab92a24f71b7aadecb0b45af82f813 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 6 Sep 2024 17:50:16 +0200 Subject: [PATCH 053/128] Add nodeSelector to Helm --- config/gesis-stage.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index 24f0da107..febc1f007 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -1,5 +1,10 @@ url: https://notebooks-test.gesis.org/binder/ +userNodeSelector: &userNodeSelector + jupyterhub_single_user: true +coreNodeSelector: &coreNodeSelector + ingress: true + binderhub: replicas: 1 @@ -14,6 +19,8 @@ binderhub: LaunchQuota: total_quota: 30 + nodeSelector: *coreNodeSelector + extraConfig: 02-badge-base-url: | c.BinderHub.badge_base_url = "https://mybinder.org/" @@ -33,6 +40,12 @@ binderhub: imageGCThresholdLow: 50e9 imageGCThresholdType: absolute + jupyterhub: + singleuser: + nodeSelector: *userNodeSelector + hub: + nodeSelector: *coreNodeSelector + prometheus: enabled: true server: From 46798b3fc8dcc28444118ddfa9f1a7661c621644 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 20 Sep 2024 14:59:14 +0200 Subject: [PATCH 054/128] Temporarlily remove svko-css-backup-node --- ansible/inventories/gesis-stage | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ansible/inventories/gesis-stage b/ansible/inventories/gesis-stage index a09315bf8..03c5e9e5e 100644 --- a/ansible/inventories/gesis-stage +++ b/ansible/inventories/gesis-stage @@ -1,6 +1,6 @@ [all] #svko-ilcm04 ansible_host=194.95.75.14 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_14 }}' -svko-css-backup-node ansible_host=194.95.75.20 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_20 }}' +; svko-css-backup-node ansible_host=194.95.75.20 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_20 }}' svko-k8s-test01 ansible_host=194.95.75.21 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_21 }}' svko-k8s-test02 ansible_host=194.95.75.22 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_22 }}' svko-k8s-test03 ansible_host=194.95.75.23 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_23 }}' @@ -11,22 +11,24 @@ K8S_CONTROL_PLANE_ENDPOINT=194.95.75.21 K8S_CONTROL_PLANE_ALIAS=svko-k8s-test01 [notebooks_gesis_org] -svko-css-backup-node +; svko-css-backup-node +svko-k8s-test02 [kubernetes_control_panel] svko-k8s-test01 [kubernetes_workers] #svko-ilcm04 -svko-css-backup-node +; svko-css-backup-node svko-k8s-test02 svko-k8s-test03 [ingress] -svko-css-backup-node +; svko-css-backup-node +svko-k8s-test02 [harbor] -svko-css-backup-node +; svko-css-backup-node [binderhub] svko-k8s-test02 @@ -35,7 +37,7 @@ svko-k8s-test02 svko-k8s-test03 [prometheus] -svko-css-backup-node +; svko-css-backup-node [grafana] -svko-css-backup-node +; svko-css-backup-node From e751c6ac633cccc626322efd56fdf9d038f543cf Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 20 Sep 2024 15:00:51 +0200 Subject: [PATCH 055/128] Isolate Container Network Interface configuration --- ansible/roles/k8s-control-panel/tasks/cni.yml | 16 ++++++++++++++++ .../roles/k8s-control-panel/tasks/main.yml | 19 +++---------------- 2 files changed, 19 insertions(+), 16 deletions(-) create mode 100644 ansible/roles/k8s-control-panel/tasks/cni.yml diff --git a/ansible/roles/k8s-control-panel/tasks/cni.yml b/ansible/roles/k8s-control-panel/tasks/cni.yml new file mode 100644 index 000000000..3763992e3 --- /dev/null +++ b/ansible/roles/k8s-control-panel/tasks/cni.yml @@ -0,0 +1,16 @@ +- name: Remove Container Network Interface (CNI) Flannel + kubernetes.core.k8s: + state: absent + src: https://github.com/coreos/flannel/raw/master/Documentation/kube-flannel.yml +- name: Install Container Network Interface (CNI) Tigera Calico operator + kubernetes.core.k8s: + state: absent + src: https://raw.githubusercontent.com/projectcalico/calico/v3.26.3/manifests/tigera-operator.yaml +- name: Install Calico and resource + kubernetes.core.k8s: + state: absent + definition: "{{ lookup('file', '{{ role_path }}/files/calico/custom-resources.yaml') | from_yaml_all }}" +- name: Install Cert Manager + kubernetes.core.k8s: + state: present + src: https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.crds.yaml diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index c4f30f0c4..fa0bbeda3 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -74,22 +74,9 @@ ansible.builtin.fetch: src: /tmp/kubernetes_join_command dest: '{{ ANSIBLE_CONTROL_NODE_TMP }}' -- name: Remove Container Network Interface (CNI) Flannel - kubernetes.core.k8s: - state: absent - src: https://github.com/coreos/flannel/raw/master/Documentation/kube-flannel.yml -- name: Install Container Network Interface (CNI) Tigera Calico operator - kubernetes.core.k8s: - state: present - src: https://raw.githubusercontent.com/projectcalico/calico/v3.26.3/manifests/tigera-operator.yaml -- name: Install Calico and resource - kubernetes.core.k8s: - state: present - definition: "{{ lookup('file', '{{ role_path }}/files/calico/custom-resources.yaml') | from_yaml_all }}" -- name: Install Cert Manager - kubernetes.core.k8s: - state: present - src: https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.crds.yaml +- name: Add Container Network Interface (CNI) to Kubernetes cluster + ansible.builtin.import_tasks: + file: cni.yml - name: Add GitLab Helm repository kubernetes.core.helm_repository: name: gitlab From 9555f59ee0340b7cad75013b471b9aaa22cb1c09 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 10:29:15 +0200 Subject: [PATCH 056/128] Improve version of Kubernetes Closes https://git.gesis.org/methods-hub/interactive-environment/-/issues/11 --- ansible/roles/k8s-common/tasks/main.yml | 18 ++++++++++++------ ansible/roles/k8s-common/vars/main.yml | 1 + 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 ansible/roles/k8s-common/vars/main.yml diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index 5495cf645..a16d6cdf6 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -20,9 +20,15 @@ state: absent - name: Remove old Kubernetes repository ansible.builtin.apt_repository: - repo: "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes.asc] https://pkgs.k8s.io/core:/stable:/v1.27/deb/ /" + repo: "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes.asc] https://pkgs.k8s.io/core:/stable:/v{{ item }}/deb/ /" filename: kubernetes state: absent + loop: + - '1.27' + - '1.28' + - '1.29' + - '1.30' + - '1.31' - name: Ensure DOCKER_CLIENT_TIMEOUT is set ansible.builtin.lineinfile: path: /etc/environment @@ -137,7 +143,7 @@ state: present - name: Download Kubernetes public GPG key ansible.builtin.get_url: - url: https://pkgs.k8s.io/core:/stable:/v1.27/deb/Release.key + url: https://pkgs.k8s.io/core:/stable:/v {{ k8s_common_kubernetes_version }}/deb/Release.key dest: /tmp/kubernetes-archive-keyring.asc mode: '0644' force: true @@ -159,7 +165,7 @@ mode: '0644' - name: Add Kubernetes repository ansible.builtin.apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/kubernetes.gpg] https://pkgs.k8s.io/core:/stable:/v1.27/deb/ /" + repo: "deb [signed-by=/etc/apt/keyrings/kubernetes.gpg] https://pkgs.k8s.io/core:/stable:/v{{ k8s_common_kubernetes_version }}/deb/ /" filename: kubernetes state: present - name: Add Helm public GPG key @@ -186,9 +192,9 @@ - ca-certificates - curl - containerd.io=1.7.* - - kubelet=1.28.* - - kubeadm=1.28.* - - kubectl=1.28.* + - "kubelet={{ k8s_common_kubernetes_version }}.*" + - "kubeadm={{ k8s_common_kubernetes_version }}.*" + - "kubectl={{ k8s_common_kubernetes_version }}.*" - helm=3.15.* - name: Copy containerd configuration file ansible.builtin.copy: diff --git a/ansible/roles/k8s-common/vars/main.yml b/ansible/roles/k8s-common/vars/main.yml new file mode 100644 index 000000000..4bfa6bc8f --- /dev/null +++ b/ansible/roles/k8s-common/vars/main.yml @@ -0,0 +1 @@ +k8s_common_kubernetes_version: "1.31" From 3e2b781cbadb466f6b8602425eadd3b0988343d9 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 10:35:53 +0200 Subject: [PATCH 057/128] Fix typo --- ansible/roles/k8s-common/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index a16d6cdf6..20c325e0a 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -143,7 +143,7 @@ state: present - name: Download Kubernetes public GPG key ansible.builtin.get_url: - url: https://pkgs.k8s.io/core:/stable:/v {{ k8s_common_kubernetes_version }}/deb/Release.key + url: https://pkgs.k8s.io/core:/stable:/v{{ k8s_common_kubernetes_version }}/deb/Release.key dest: /tmp/kubernetes-archive-keyring.asc mode: '0644' force: true From 8385101998f5306ac83aedf97e95b857dd5de7ed Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 11:05:09 +0200 Subject: [PATCH 058/128] Upgrade Helm version --- ansible/roles/k8s-common/tasks/main.yml | 2 +- ansible/roles/k8s-common/vars/main.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index 20c325e0a..321f243a1 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -195,7 +195,7 @@ - "kubelet={{ k8s_common_kubernetes_version }}.*" - "kubeadm={{ k8s_common_kubernetes_version }}.*" - "kubectl={{ k8s_common_kubernetes_version }}.*" - - helm=3.15.* + - "helm={{ k8s_common_helm_version }}" - name: Copy containerd configuration file ansible.builtin.copy: src: files/etc/containerd/config.toml diff --git a/ansible/roles/k8s-common/vars/main.yml b/ansible/roles/k8s-common/vars/main.yml index 4bfa6bc8f..6bfebd61c 100644 --- a/ansible/roles/k8s-common/vars/main.yml +++ b/ansible/roles/k8s-common/vars/main.yml @@ -1 +1,2 @@ k8s_common_kubernetes_version: "1.31" +k8s_common_helm_version: "3.16.1" From 369dadca074e96009939650cae4323f5fe0a08b5 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 11:10:45 +0200 Subject: [PATCH 059/128] Fix version matching --- ansible/roles/k8s-common/tasks/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index 321f243a1..09d9ead14 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -192,10 +192,10 @@ - ca-certificates - curl - containerd.io=1.7.* - - "kubelet={{ k8s_common_kubernetes_version }}.*" - - "kubeadm={{ k8s_common_kubernetes_version }}.*" - - "kubectl={{ k8s_common_kubernetes_version }}.*" - - "helm={{ k8s_common_helm_version }}" + - "kubelet={{ k8s_common_kubernetes_version }}*" + - "kubeadm={{ k8s_common_kubernetes_version }}*" + - "kubectl={{ k8s_common_kubernetes_version }}*" + - "helm={{ k8s_common_helm_version }}*" - name: Copy containerd configuration file ansible.builtin.copy: src: files/etc/containerd/config.toml From ec82162729f14c0eb75f630349a91c8d0a680620 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 11:18:23 +0200 Subject: [PATCH 060/128] Restore Calico --- ansible/roles/k8s-control-panel/tasks/cni.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/k8s-control-panel/tasks/cni.yml b/ansible/roles/k8s-control-panel/tasks/cni.yml index 3763992e3..73db6453a 100644 --- a/ansible/roles/k8s-control-panel/tasks/cni.yml +++ b/ansible/roles/k8s-control-panel/tasks/cni.yml @@ -4,11 +4,11 @@ src: https://github.com/coreos/flannel/raw/master/Documentation/kube-flannel.yml - name: Install Container Network Interface (CNI) Tigera Calico operator kubernetes.core.k8s: - state: absent + state: present src: https://raw.githubusercontent.com/projectcalico/calico/v3.26.3/manifests/tigera-operator.yaml - name: Install Calico and resource kubernetes.core.k8s: - state: absent + state: present definition: "{{ lookup('file', '{{ role_path }}/files/calico/custom-resources.yaml') | from_yaml_all }}" - name: Install Cert Manager kubernetes.core.k8s: From 89639cc30d8609dd5043ad6164611ab66f498ab2 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 11:22:32 +0200 Subject: [PATCH 061/128] Upgrade Calico --- .../files/calico/custom-resources.yaml | 18 ++++++++++-------- ansible/roles/k8s-control-panel/tasks/cni.yml | 3 ++- ansible/roles/k8s-control-panel/vars/main.yml | 1 + 3 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 ansible/roles/k8s-control-panel/vars/main.yml diff --git a/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml b/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml index 4a90b5bc2..dfb43860b 100644 --- a/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml +++ b/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml @@ -1,5 +1,5 @@ # This section includes base Calico installation configuration. -# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation +# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.Installation apiVersion: operator.tigera.io/v1 kind: Installation metadata: @@ -7,16 +7,18 @@ metadata: spec: # Configures Calico networking. calicoNetwork: - # Note: The ipPools section cannot be modified post-install. ipPools: - - blockSize: 26 - cidr: 10.244.0.0/16 - encapsulation: VXLANCrossSubnet - natOutgoing: Enabled - nodeSelector: all() + - name: default-ipv4-ippool + blockSize: 26 + cidr: 192.168.0.0/16 + encapsulation: VXLANCrossSubnet + natOutgoing: Enabled + nodeSelector: all() + --- + # This section configures the Calico API server. -# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer +# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.APIServer apiVersion: operator.tigera.io/v1 kind: APIServer metadata: diff --git a/ansible/roles/k8s-control-panel/tasks/cni.yml b/ansible/roles/k8s-control-panel/tasks/cni.yml index 73db6453a..7a091d9e3 100644 --- a/ansible/roles/k8s-control-panel/tasks/cni.yml +++ b/ansible/roles/k8s-control-panel/tasks/cni.yml @@ -5,10 +5,11 @@ - name: Install Container Network Interface (CNI) Tigera Calico operator kubernetes.core.k8s: state: present - src: https://raw.githubusercontent.com/projectcalico/calico/v3.26.3/manifests/tigera-operator.yaml + src: https://raw.githubusercontent.com/projectcalico/calico/v{{ k8s_control_panel_calico_version }}/manifests/tigera-operator.yaml - name: Install Calico and resource kubernetes.core.k8s: state: present + # A local copy of https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/custom-resources.yaml definition: "{{ lookup('file', '{{ role_path }}/files/calico/custom-resources.yaml') | from_yaml_all }}" - name: Install Cert Manager kubernetes.core.k8s: diff --git a/ansible/roles/k8s-control-panel/vars/main.yml b/ansible/roles/k8s-control-panel/vars/main.yml new file mode 100644 index 000000000..daa8e21d2 --- /dev/null +++ b/ansible/roles/k8s-control-panel/vars/main.yml @@ -0,0 +1 @@ +k8s_control_panel_calico_version: "3.28.2" From 9b52694302dbd1eb6faeef2836256b7e97416a49 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 11:24:29 +0200 Subject: [PATCH 062/128] Fix wrong indentation --- .../files/calico/custom-resources.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml b/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml index dfb43860b..039bf33ca 100644 --- a/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml +++ b/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml @@ -8,12 +8,12 @@ spec: # Configures Calico networking. calicoNetwork: ipPools: - - name: default-ipv4-ippool - blockSize: 26 - cidr: 192.168.0.0/16 - encapsulation: VXLANCrossSubnet - natOutgoing: Enabled - nodeSelector: all() + - name: default-ipv4-ippool + blockSize: 26 + cidr: 192.168.0.0/16 + encapsulation: VXLANCrossSubnet + natOutgoing: Enabled + nodeSelector: all() --- From ede919611e97cc638d5a615280d6364c7e61cae0 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 11:32:53 +0200 Subject: [PATCH 063/128] Fix cannot unmarshal bool into Go struct field --- config/gesis-stage.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index febc1f007..bbf741cf6 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -1,9 +1,9 @@ url: https://notebooks-test.gesis.org/binder/ userNodeSelector: &userNodeSelector - jupyterhub_single_user: true + jupyterhub_single_user: "true" coreNodeSelector: &coreNodeSelector - ingress: true + ingress: "true" binderhub: replicas: 1 From df421012ae1ba333567810f41127666c619ccbfa Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 11:43:46 +0200 Subject: [PATCH 064/128] Comment Tigera --- .gitlab-ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1702c3a2e..11ccfba84 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,16 +40,16 @@ include: for d in ./mybinder*/; do helm dependency update "$d"; done - - | - for chart in mybinder-kube-system mybinder-tigera-operator; do - helm upgrade \ - ${chart:9} ./${chart} \ - --cleanup-on-fail \ - --create-namespace \ - --history-max 3 \ - --install \ - --namespace=${chart}; - done + # - | + # for chart in mybinder-kube-system mybinder-tigera-operator; do + # helm upgrade \ + # ${chart:9} ./${chart} \ + # --cleanup-on-fail \ + # --create-namespace \ + # --history-max 3 \ + # --install \ + # --namespace=${chart}; + # done - | helm lint ./mybinder \ --values ./config/gesis-${HELM_ENVIRONMENT}.yaml From 99de37e8e318b03c6f120105bc1f732c10a4695c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 13:06:42 +0200 Subject: [PATCH 065/128] Deploy same helm chart twice --- ansible/roles/k8s-control-panel/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index fa0bbeda3..58b8b169b 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -83,7 +83,7 @@ repo_url: https://charts.gitlab.io - name: Deploy GitLab agent kubernetes.core.helm: - name: gitlab-agent + name: 'gitlab-agent-{{ ansible_hostname }}' chart_ref: gitlab/gitlab-agent release_namespace: gitlab-agent dependency_update: true From c5057b0a6c4dac98d56b767a1830625078ffc3dc Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 13:09:28 +0200 Subject: [PATCH 066/128] Fix typo --- ansible/roles/k8s-control-panel/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 58b8b169b..cbab8ee12 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -83,7 +83,7 @@ repo_url: https://charts.gitlab.io - name: Deploy GitLab agent kubernetes.core.helm: - name: 'gitlab-agent-{{ ansible_hostname }}' + name: 'gitlab-agent-{{ ansible_hostname }}' chart_ref: gitlab/gitlab-agent release_namespace: gitlab-agent dependency_update: true From 0363cc29f9f038f0e973c5bc8400dc8a01cdd49c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 13:13:19 +0200 Subject: [PATCH 067/128] Use inventory name for GitLab agent --- ansible/roles/k8s-control-panel/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index cbab8ee12..4b14de201 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -83,7 +83,7 @@ repo_url: https://charts.gitlab.io - name: Deploy GitLab agent kubernetes.core.helm: - name: 'gitlab-agent-{{ ansible_hostname }}' + name: 'gitlab-agent-{{ inventory_file | basename }}' chart_ref: gitlab/gitlab-agent release_namespace: gitlab-agent dependency_update: true From 53307a0a3855bced8b8970638a0c9cf466796f60 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 13:48:33 +0200 Subject: [PATCH 068/128] Fix Calico IP pools --- .../roles/k8s-control-panel/files/calico/custom-resources.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml b/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml index 039bf33ca..746e1d94d 100644 --- a/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml +++ b/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml @@ -10,7 +10,7 @@ spec: ipPools: - name: default-ipv4-ippool blockSize: 26 - cidr: 192.168.0.0/16 + cidr: 10.244.0.0/16 encapsulation: VXLANCrossSubnet natOutgoing: Enabled nodeSelector: all() From ab28076e8b1322cd498eece326fed689b57d42ff Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 16:03:20 +0200 Subject: [PATCH 069/128] Run chartpress --- .gitlab-ci.yml | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11ccfba84..75cf3768b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,27 @@ include: ssh-user: ansible ssh-key-type: ed25519 +Create Helm release number: + stage: build + image: python:3.12.6-alpine3.20 + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + cache: + paths: + - .cache/pip + script: + - python -V + - python -m pip install virtualenv + - virtualenv venv + - source venv/bin/activate + - python -m pip install chartpress + - chartpress + - cp -r mybinder dist + artifacts: + paths: + - dist/ + expire_in: 1h + .gesis helm deploy: image: name: docker-private.gesis.intra/gesis/ilcm/orc2/k8s:latest @@ -37,7 +58,7 @@ include: - kubectl config use-context ${CI_PROJECT_PATH}:${HELM_ENVIRONMENT} - helm version - | - for d in ./mybinder*/; do + for d in ./dist*/; do helm dependency update "$d"; done # - | @@ -51,11 +72,14 @@ include: # --namespace=${chart}; # done - | - helm lint ./mybinder \ - --values ./config/gesis-${HELM_ENVIRONMENT}.yaml + helm lint ./dist \ + --values ./config/gesis-${HELM_ENVIRONMENT}.yaml \ + --values ./secrets/config/common/common.yaml \ + --values ./secrets/config/common/cryptnono.yaml \ + --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml - | helm upgrade \ - binderhub ./mybinder \ + binderhub ./dist \ --cleanup-on-fail \ --create-namespace \ --history-max 3 \ From 7eed54ab5d4c47c831830e746a0e654033e5e951 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 16:16:18 +0200 Subject: [PATCH 070/128] Change to use Alpine --- .gitlab-ci.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75cf3768b..b4acaea2b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,19 +27,17 @@ include: Create Helm release number: stage: build - image: python:3.12.6-alpine3.20 + image: alpine:3.20.3 variables: + APK_CACHE_DIR: "$CI_PROJECT_DIR/.cache/apk" PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" cache: paths: - - .cache/pip + - .cache/ script: - - python -V - - python -m pip install virtualenv - - virtualenv venv - - source venv/bin/activate - - python -m pip install chartpress - - chartpress + - apk add python3 pipx + - pipx install chartpress + - pipx run chartpress - cp -r mybinder dist artifacts: paths: From a379b89598f240b12e79e9e87522771e661e1a16 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 16:17:12 +0200 Subject: [PATCH 071/128] Install missing Git to Alpine --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4acaea2b..482392686 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ Create Helm release number: paths: - .cache/ script: - - apk add python3 pipx + - apk add git python3 pipx - pipx install chartpress - pipx run chartpress - cp -r mybinder dist From 089718dc5474320cb9049c7e68e35b411c499386 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 16:19:53 +0200 Subject: [PATCH 072/128] Add missing Docker to Alpine --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 482392686..ed55ae42c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ Create Helm release number: paths: - .cache/ script: - - apk add git python3 pipx + - apk add git docker python3 pipx - pipx install chartpress - pipx run chartpress - cp -r mybinder dist From 2a30931509b81138724612d45b2e404c15048505 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 16:22:30 +0200 Subject: [PATCH 073/128] Add missing helm --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed55ae42c..954bbc663 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ Create Helm release number: paths: - .cache/ script: - - apk add git docker python3 pipx + - apk add git docker helm python3 pipx - pipx install chartpress - pipx run chartpress - cp -r mybinder dist From 51535cf2e14a6d9634b7e71420ed3b4e01339bc0 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 16:28:05 +0200 Subject: [PATCH 074/128] Add Docker in Docker service --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 954bbc663..cd26201dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,8 @@ include: Create Helm release number: stage: build image: alpine:3.20.3 + services: + - docker:dind variables: APK_CACHE_DIR: "$CI_PROJECT_DIR/.cache/apk" PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -35,7 +37,7 @@ Create Helm release number: paths: - .cache/ script: - - apk add git docker helm python3 pipx + - apk add git helm python3 pipx - pipx install chartpress - pipx run chartpress - cp -r mybinder dist From 7087dd297241c1038d7905158e404d9ac687517f Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 16:31:40 +0200 Subject: [PATCH 075/128] Revert creation of release number --- .gitlab-ci.yml | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd26201dc..baf4b2a44 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,27 +25,6 @@ include: ssh-user: ansible ssh-key-type: ed25519 -Create Helm release number: - stage: build - image: alpine:3.20.3 - services: - - docker:dind - variables: - APK_CACHE_DIR: "$CI_PROJECT_DIR/.cache/apk" - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - cache: - paths: - - .cache/ - script: - - apk add git helm python3 pipx - - pipx install chartpress - - pipx run chartpress - - cp -r mybinder dist - artifacts: - paths: - - dist/ - expire_in: 1h - .gesis helm deploy: image: name: docker-private.gesis.intra/gesis/ilcm/orc2/k8s:latest @@ -58,7 +37,7 @@ Create Helm release number: - kubectl config use-context ${CI_PROJECT_PATH}:${HELM_ENVIRONMENT} - helm version - | - for d in ./dist*/; do + for d in ./mybinder*/; do helm dependency update "$d"; done # - | @@ -72,7 +51,7 @@ Create Helm release number: # --namespace=${chart}; # done - | - helm lint ./dist \ + helm lint ./mybinder \ --values ./config/gesis-${HELM_ENVIRONMENT}.yaml \ --values ./secrets/config/common/common.yaml \ --values ./secrets/config/common/cryptnono.yaml \ From ab77b7f6a9e612cda9c0e60a5bfb39f8b17fbdd8 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 16:35:01 +0200 Subject: [PATCH 076/128] Fix typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index baf4b2a44..85bf3b0cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ include: --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml - | helm upgrade \ - binderhub ./dist \ + binderhub ./mybinder \ --cleanup-on-fail \ --create-namespace \ --history-max 3 \ From b6144d1ea8e0875ce6188ca9a55d37c149c2a9de Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 23 Sep 2024 16:44:10 +0200 Subject: [PATCH 077/128] Fix CLOUD_SDK_MISSING_CREDENTIALS --- config/gesis-stage.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index bbf741cf6..d44f0ab1e 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -5,6 +5,9 @@ userNodeSelector: &userNodeSelector coreNodeSelector: &coreNodeSelector ingress: "true" +analyticsPublisher: + enabled: false + binderhub: replicas: 1 From 54b95107bd79daac0f22ae21dabbadb1ca4cd61e Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Wed, 11 Sep 2024 13:31:09 +0200 Subject: [PATCH 078/128] Avoid Helm deployment --- .gitlab-ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 85bf3b0cb..30c7537a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,15 +70,15 @@ include: --values ./secrets/config/common/cryptnono.yaml \ --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml -gesis helm stage deploy: - resource_group: stage - stage: deploy-stage-helm - variables: - HELM_ENVIRONMENT: stage - extends: - - .gesis helm deploy +# gesis helm stage deploy: +# resource_group: stage +# stage: deploy-stage-helm +# variables: +# HELM_ENVIRONMENT: stage +# extends: +# - .gesis helm deploy -smoke test to stage cluster: - stage: test-stage - script: - - curl https://notebooks-test.gesis.org/binder/ \ No newline at end of file +# smoke test to stage cluster: +# stage: test-stage +# script: +# - curl https://notebooks-test.gesis.org/binder/ \ No newline at end of file From 0e1cd7c7e13b9f42f5fcd65835007eaaf8615564 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Wed, 11 Sep 2024 13:42:24 +0200 Subject: [PATCH 079/128] Upgrade Kubernetes --- .../roles/k8s-common/tasks/k8s-repository.yml | 51 ++++++++++++++++++ ansible/roles/k8s-common/tasks/main.yml | 54 ++----------------- 2 files changed, 54 insertions(+), 51 deletions(-) create mode 100644 ansible/roles/k8s-common/tasks/k8s-repository.yml diff --git a/ansible/roles/k8s-common/tasks/k8s-repository.yml b/ansible/roles/k8s-common/tasks/k8s-repository.yml new file mode 100644 index 000000000..2fc66d81b --- /dev/null +++ b/ansible/roles/k8s-common/tasks/k8s-repository.yml @@ -0,0 +1,51 @@ +- name: Remove old Kubernetes public GPG key + ansible.builtin.file: + path: /etc/apt/trusted.gpg.d/kubernetes-archive-keyring.gpg + state: absent +- name: Remove old Kubernetes public GPG key + ansible.builtin.file: + path: /etc/apt/trusted.gpg.d/kubernetes-archive-keyring.asc + state: absent +- name: Remove old Kubernetes repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" + filename: kubernetes + state: absent +- name: Remove old Kubernetes repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes.asc] https://pkgs.k8s.io/core:/stable:/{{ item }}/deb/ /" + filename: kubernetes + state: absent + loop: + - v1.27 + - v1.28 + - v1.29 + - v1.30 + - v1.31 +- name: Download Kubernetes public GPG key + ansible.builtin.get_url: + url: "https://pkgs.k8s.io/core:/stable:/v{{ k8s_common_kubernetes_version }}/deb/Release.key" + dest: /tmp/kubernetes-archive-keyring.asc + mode: '0644' + force: true +- name: Convert the public GPG key to binary + ansible.builtin.command: + argv: + - gpg + - --yes + - --dearmor + - --output + - /tmp/kubernetes.gpg + - /tmp/kubernetes-archive-keyring.asc + changed_when: false +- name: Copy GPG key + ansible.builtin.copy: + src: /tmp/kubernetes.gpg + dest: /etc/apt/keyrings/kubernetes.gpg + remote_src: true + mode: '0644' +- name: Add Kubernetes repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/keyrings/kubernetes.gpg] https://pkgs.k8s.io/core:/stable:/v{{ k8s_common_kubernetes_version }}/deb/ /" + filename: kubernetes + state: present diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index 09d9ead14..da40ca290 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -5,30 +5,9 @@ owner: root group: root mode: u=rwx,g=rx,o=rx -- name: Remove old Kubernetes public GPG key - ansible.builtin.file: - path: /etc/apt/trusted.gpg.d/kubernetes-archive-keyring.gpg - state: absent -- name: Remove old Kubernetes public GPG key - ansible.builtin.file: - path: /etc/apt/trusted.gpg.d/kubernetes-archive-keyring.asc - state: absent -- name: Remove old Kubernetes repository - ansible.builtin.apt_repository: - repo: "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" - filename: kubernetes - state: absent -- name: Remove old Kubernetes repository - ansible.builtin.apt_repository: - repo: "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes.asc] https://pkgs.k8s.io/core:/stable:/v{{ item }}/deb/ /" - filename: kubernetes - state: absent - loop: - - '1.27' - - '1.28' - - '1.29' - - '1.30' - - '1.31' +- name: Add Kubernetes repository + ansible.builtin.import_tasks: + file: k8s-repository.yml - name: Ensure DOCKER_CLIENT_TIMEOUT is set ansible.builtin.lineinfile: path: /etc/environment @@ -141,33 +120,6 @@ repo: "deb [signed-by=/etc/apt/trusted.gpg.d/docker.asc] https://download.docker.com/linux/ubuntu jammy stable" filename: docker state: present -- name: Download Kubernetes public GPG key - ansible.builtin.get_url: - url: https://pkgs.k8s.io/core:/stable:/v{{ k8s_common_kubernetes_version }}/deb/Release.key - dest: /tmp/kubernetes-archive-keyring.asc - mode: '0644' - force: true -- name: Convert the public GPG key to binary - ansible.builtin.command: - argv: - - gpg - - --yes - - --dearmor - - --output - - /tmp/kubernetes.gpg - - /tmp/kubernetes-archive-keyring.asc - changed_when: false -- name: Copy GPG key - ansible.builtin.copy: - src: /tmp/kubernetes.gpg - dest: /etc/apt/keyrings/kubernetes.gpg - remote_src: true - mode: '0644' -- name: Add Kubernetes repository - ansible.builtin.apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/kubernetes.gpg] https://pkgs.k8s.io/core:/stable:/v{{ k8s_common_kubernetes_version }}/deb/ /" - filename: kubernetes - state: present - name: Add Helm public GPG key ansible.builtin.get_url: url: https://baltocdn.com/helm/signing.asc From 66906dbca0a313f49ef61582ad816a51cd114b70 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 09:37:36 +0200 Subject: [PATCH 080/128] Add configuration files from Pixi --- .gitattributes | 2 ++ .gitignore | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.gitattributes b/.gitattributes index 2376958bf..9208e10ac 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ **/secrets/** filter=git-crypt diff=git-crypt +# GitHub syntax highlighting +pixi.lock linguist-language=YAML linguist-generated=true diff --git a/.gitignore b/.gitignore index f62deeeb4..3f0fa6888 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ travis/crypt-key env .terraform +# pixi environments +.pixi +*.egg-info From f9bb594511eb33971c0f676d6c45ef88ad0d0b16 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 09:37:55 +0200 Subject: [PATCH 081/128] Add variables to inventory --- ansible/inventories/gesis-stage | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/inventories/gesis-stage b/ansible/inventories/gesis-stage index 03c5e9e5e..66beeb4d3 100644 --- a/ansible/inventories/gesis-stage +++ b/ansible/inventories/gesis-stage @@ -17,6 +17,11 @@ svko-k8s-test02 [kubernetes_control_panel] svko-k8s-test01 +[kubernetes_control_panel:vars] +GRAFANA_CAPACITY_STORAGE=2Gi +JUPYTERHUB_CAPACITY_STORAGE=2Gi +PROMETHEUS_CAPACITY_STORAGE=15Gi + [kubernetes_workers] #svko-ilcm04 ; svko-css-backup-node @@ -33,6 +38,9 @@ svko-k8s-test02 [binderhub] svko-k8s-test02 +[jupyterhub] +svko-k8s-test02 + [jupyterhub_single_user] svko-k8s-test03 From ca5a800da524871e35bc3c2c14740852d5734d51 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 09:52:28 +0200 Subject: [PATCH 082/128] Checkout changes from older branch --- ansible/gesis.yml | 6 + ansible/roles/k8s-common/tasks/main.yml | 73 +------ ansible/roles/k8s-common/tasks/pv.yml | 28 +++ ansible/roles/k8s-pv/tasks/grafana.yml | 29 +++ ansible/roles/k8s-pv/tasks/jupyter-hub-db.yml | 29 +++ ansible/roles/k8s-pv/tasks/main.yml | 22 ++ ansible/roles/k8s-pv/tasks/prometheus.yml | 29 +++ config/gesis-stage.yaml | 199 ++++++++++++------ mybinder/templates/ingress-nginx/ingress.yaml | 24 --- secrets/config/common/gesis.yaml | Bin 0 -> 4529 bytes secrets/config/gesis-stage.yaml | Bin 10989 -> 11420 bytes 11 files changed, 277 insertions(+), 162 deletions(-) create mode 100644 ansible/roles/k8s-common/tasks/pv.yml create mode 100644 ansible/roles/k8s-pv/tasks/grafana.yml create mode 100644 ansible/roles/k8s-pv/tasks/jupyter-hub-db.yml create mode 100644 ansible/roles/k8s-pv/tasks/main.yml create mode 100644 ansible/roles/k8s-pv/tasks/prometheus.yml delete mode 100644 mybinder/templates/ingress-nginx/ingress.yaml create mode 100644 secrets/config/common/gesis.yaml diff --git a/ansible/gesis.yml b/ansible/gesis.yml index 66b92df7d..430e341b0 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -16,6 +16,12 @@ become: true roles: - k8s-worker +- name: Configure Kubernetes Persistent Volumes + hosts: kubernetes_control_panel + gather_facts: false + become: true + roles: + - k8s-pv - name: Configure JupyterHub workers hosts: jupyterhub_single_user gather_facts: false diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index da40ca290..e8aac97b6 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -39,76 +39,9 @@ name: fs.inotify.max_user_watches value: '655360' state: present -- name: Create /orc2_data/containerd directory if it does not exist - ansible.builtin.file: - path: /orc2_data/containerd - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx -- name: Create /orc2_data/repo2docker directory if it does not exist - ansible.builtin.file: - path: /orc2_data/repo2docker - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx -- name: Create /orc2_data/prometheus directory if it does not exist - ansible.builtin.file: - path: /orc2_data/prometheus - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx -- name: Create /orc2_data/grafana directory if it does not exist - ansible.builtin.file: - path: /orc2_data/grafana - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx -- name: Create /orc2_data/alertmanager directory if it does not exist - ansible.builtin.file: - path: /orc2_data/alertmanager - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx -- name: Create /harbor/jobservice directory if it does not exist - ansible.builtin.file: - path: /harbor/jobservice - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx -- name: Create /harbor/registry directory if it does not exist - ansible.builtin.file: - path: /harbor/registry - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx -- name: Create /harbor/redis directory if it does not exist - ansible.builtin.file: - path: /harbor/redis - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx -- name: Create /harbor/trivy directory if it does not exist - ansible.builtin.file: - path: /harbor/trivy - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx -- name: Create /harbor/database directory if it does not exist - ansible.builtin.file: - path: /harbor/database - state: directory - owner: root - group: root - mode: u=rwx,g=rx,o=rx +- name: Create directory for Persistent Volume + ansible.builtin.import_tasks: + file: pv.yml - name: Add Docker public GPG key ansible.builtin.get_url: url: https://download.docker.com/linux/ubuntu/gpg diff --git a/ansible/roles/k8s-common/tasks/pv.yml b/ansible/roles/k8s-common/tasks/pv.yml new file mode 100644 index 000000000..6a11ee11e --- /dev/null +++ b/ansible/roles/k8s-common/tasks/pv.yml @@ -0,0 +1,28 @@ +- name: Create persistent directories in /orc2_data if it does not exist + ansible.builtin.file: + path: "/orc2_data/{{ item }}" + state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx + loop: + - jupyterhub + - containerd + - repo2docker + - prometheus + - grafana + - alertmanager + +- name: Create persistent directories in /harbor/ if it does not exist + ansible.builtin.file: + path: "/harbor/{{ item }}" + state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx + loop: + - jobservice + - registry + - redis + - trivy + - database diff --git a/ansible/roles/k8s-pv/tasks/grafana.yml b/ansible/roles/k8s-pv/tasks/grafana.yml new file mode 100644 index 000000000..e983576ad --- /dev/null +++ b/ansible/roles/k8s-pv/tasks/grafana.yml @@ -0,0 +1,29 @@ +- name: Create a Persistent Volume for Grafana + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolume + metadata: + name: grafana + labels: + app.kubernetes.io/managed-by: Ansible + app.kubernetes.io/part-of: grafana + spec: + capacity: + storage: "{{ GRAFANA_CAPACITY_STORAGE }}" + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: "local-storage" + local: + path: /orc2_data/grafana + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: grafana + operator: In + values: + - "true" diff --git a/ansible/roles/k8s-pv/tasks/jupyter-hub-db.yml b/ansible/roles/k8s-pv/tasks/jupyter-hub-db.yml new file mode 100644 index 000000000..55bcab192 --- /dev/null +++ b/ansible/roles/k8s-pv/tasks/jupyter-hub-db.yml @@ -0,0 +1,29 @@ +- name: Create a Persistent Volume for JupyterHub + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolume + metadata: + name: "jupyterhub-db" + labels: + app.kubernetes.io/managed-by: Ansible + app.kubernetes.io/part-of: jupyterhub + spec: + capacity: + storage: "{{ JUPYTERHUB_CAPACITY_STORAGE }}" + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: "local-storage" + local: + path: /orc2_data/jupyterhub + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: jupyterhub + operator: In + values: + - "true" diff --git a/ansible/roles/k8s-pv/tasks/main.yml b/ansible/roles/k8s-pv/tasks/main.yml new file mode 100644 index 000000000..e83a385b4 --- /dev/null +++ b/ansible/roles/k8s-pv/tasks/main.yml @@ -0,0 +1,22 @@ +- name: Create a Persistent Volume for Prometheus + kubernetes.core.k8s: + state: present + definition: + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: local-storage + labels: + app.kubernetes.io/managed-by: Ansible + app.kubernetes.io/part-of: mybinder + provisioner: kubernetes.io/no-provisioner + volumeBindingMode: WaitForFirstConsumer +- name: Provide Persistent Volume for Grafana + ansible.builtin.import_tasks: + file: grafana.yml +- name: Provide Persistent Volume for JupyterHub + ansible.builtin.import_tasks: + file: jupyter-hub-db.yml +- name: Provide Persistent Volume for Prometheus + ansible.builtin.import_tasks: + file: prometheus.yml diff --git a/ansible/roles/k8s-pv/tasks/prometheus.yml b/ansible/roles/k8s-pv/tasks/prometheus.yml new file mode 100644 index 000000000..ca4aa8a8a --- /dev/null +++ b/ansible/roles/k8s-pv/tasks/prometheus.yml @@ -0,0 +1,29 @@ +- name: Create a Persistent Volume for Prometheus + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolume + metadata: + name: prometheus + labels: + app.kubernetes.io/managed-by: Ansible + app.kubernetes.io/part-of: prometheus + spec: + capacity: + storage: "{{ PROMETHEUS_CAPACITY_STORAGE }}" + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: "local-storage" + local: + path: /orc2_data/prometheus + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: prometheus + operator: In + values: + - "true" diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index d44f0ab1e..af7970be1 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -1,106 +1,169 @@ -url: https://notebooks-test.gesis.org/binder/ - -userNodeSelector: &userNodeSelector - jupyterhub_single_user: "true" -coreNodeSelector: &coreNodeSelector - ingress: "true" - analyticsPublisher: enabled: false - binderhub: - replicas: 1 - - ingress: - hosts: - - notebooks-test.gesis.org - + nodeSelector: + ingress: "true" config: BinderHub: + base_url: /binder/ + build_node_selector: + binderhub: "true" hub_url: https://notebooks-test.gesis.org/binder/jupyter/ - + image_prefix: gesiscss/binder-r2d-g5b5b759- + template_path: /etc/binderhub/templates + use_registry: true + KubernetesBuildExecutor: + memory_limit: 3G + memory_request: 1G + node_selector: + binderhub: "true" + docker_available: "true" LaunchQuota: total_quota: 30 - - nodeSelector: *coreNodeSelector - extraConfig: - 02-badge-base-url: | - c.BinderHub.badge_base_url = "https://mybinder.org/" - 01-template-variables: | + 01-template-variables: > template_vars = { "gesis_notebooks_https": 'https://notebooks-test.gesis.org/', 'production': False, } - template_vars['gesis_notebooks_static'] = template_vars['gesis_notebooks_https'] + "static/" - template_vars['gesis_web_frontend_framework'] = template_vars['gesis_notebooks_static'] + "gesis-web-frontend-framework/" - template_vars['binder_static'] = template_vars['gesis_notebooks_https'] + "binder/static/" - c.BinderHub.template_variables.update(template_vars) + template_vars['gesis_notebooks_static'] = + template_vars['gesis_notebooks_https'] + "static/" + + template_vars['gesis_web_frontend_framework'] = + template_vars['gesis_notebooks_static'] + "gesis-web-frontend-framework/" + + template_vars['binder_static'] = template_vars['gesis_notebooks_https'] + + "binder/static/" + + c.BinderHub.template_variables.update(template_vars) + 02-badge-base-url: | + c.BinderHub.badge_base_url = "https://mybinder.org/" + extraEnv: + GOOGLE_APPLICATION_CREDENTIALS: /secrets/service-account.json + extraVolumeMounts: + - mountPath: /secrets + name: secrets + readOnly: true + extraVolumes: + - name: secrets + secret: + secretName: events-archiver-secrets imageCleaner: enabled: true imageGCThresholdHigh: 80e9 imageGCThresholdLow: 50e9 imageGCThresholdType: absolute - + ingress: + hosts: + - notebooks-test.gesis.org jupyterhub: - singleuser: - nodeSelector: *userNodeSelector hub: - nodeSelector: *coreNodeSelector - -prometheus: - enabled: true - server: - service: - type: NodePort - nodePort: 30073 - servicePort: 80 - livenessProbeInitialDelay: 800 - resources: - requests: - cpu: "1" - memory: 1Gi - limits: - cpu: "1" - memory: 1Gi - persistentVolume: - size: 10Gi - storageClass: standard - retention: 30d + baseUrl: /jupyterhub + db: + pvc: + storageClassName: local-storage + nodeSelector: + jupyterhub: "true" + singleuser: + nodeSelector: + jupyterhub_single_user: "true" ingress: hosts: - - notebooks-test.gesis.org/prometheus - + - notebooks-test.gesis.org + replicas: 1 +cryptnono: + enabled: true grafana: + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - disableDeletion: true + editable: false + folder: notebooks.gesis.org + name: default + options: + path: /var/lib/grafana/dashboards/notebooks.gesis.org + orgId: 1 + type: file + datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - editable: false + isDefault: true + name: GESIS Notebooks Prometheus + orgId: 1 + type: prometheus + uid: gesis-notebooks-prometheus + url: http://binderhub-prometheus-server + prune: true + deploymentStrategy: + type: Recreate enabled: true + grafana.ini: + auth.anonymous: + enabled: true + org_name: Main Org. + org_role: Viewer + auth.basic: + enabled: true + security: + allow_embedding: true + server: + http_port: 3000 + root_url: https://notebooks.gesis.org/grafana/ + smtp: + enabled: true + ingress: + hosts: + - notebooks-test.gesis.org + path: /grafana + nodeSelector: + grafana: "true" + persistence: + enabled: false resources: - requests: - cpu: "0" - memory: 128Mi limits: cpu: "0.25" memory: 128Mi - ingress: - path: /grafana - hosts: - - notebooks-test.gesis.org - -cryptnono: - enabled: true - + requests: + cpu: "0" + memory: 128Mi ingress-nginx: controller: + replicaCount: 1 + nodeSelector: + ingress: "true" + hostPort: + enable: true scope: enabled: true service: - type: ClusterIP externalTrafficPolicy: null - - + type: ClusterIP +prometheus: + enabled: true + server: + ingress: + hosts: + - notebooks-test.gesis.org + path: /prometheus + livenessProbeInitialDelay: 800 + persistentVolume: + size: 10Gi + storageClass: local-storage + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: "1" + memory: 1Gi + retention: 30d static: ingress: hosts: - - notebooks-test.gesis.org - tls: - secretName: kubelego-tls-static + - static.notebooks-test.gesis.org +url: https://notebooks-test.gesis.org/binder/ diff --git a/mybinder/templates/ingress-nginx/ingress.yaml b/mybinder/templates/ingress-nginx/ingress.yaml deleted file mode 100644 index fbdac38e7..000000000 --- a/mybinder/templates/ingress-nginx/ingress.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- $ingressType := index .Values "ingress-nginx" "controller" "service" "type" }} -{{- if eq $ingressType "ClusterIP" }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ .Release.Name }} - annotations: - nginx.ingress.kubernetes.io/rewrite-target: / -spec: - ingressClassName: {{ .Release.Name }} - rules: - {{- range .Values.binderhub.ingress.hosts }} - - host: {{ . }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: binder - port: - number: 80 - {{- end }} -{{- end }} \ No newline at end of file diff --git a/secrets/config/common/gesis.yaml b/secrets/config/common/gesis.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a62d2ecc6b0c77ab1df7bb00cbb14f4513163caa GIT binary patch literal 4529 zcmV;i5l-#^M@dveQdv+`0D=%e{i)BF#Zt;{Ss%1s12jCF2&~4L=Y)zrfi{4W4S}&6 z+Q^w9{pulFs)1|=f`9Q_dqOgp*y~?_@-G!liSH_d?Tnl8p$ZyoL5egI#1i$v*AQuW z!v4VZ4pAWQT}sI28%+BCgo2Y8ctwX9Vr+ZLT)v(bE|D>-xqv4Z*YcidAib(~c;6LMNh}VE=OCv;!g` zFN`XE+Lz{^d)lauy+M;ItEy$`MX@Mc#m1iXHAWcIw9sM1k)z4!#LaxQlHJh4Ky@nz z>)_e?zB?#7GjxL(^e~DCW74SXb4TgkXr?_`7+P5WPXQBL3O(V-Sc={^#YPYX$c*mI z*#XRJA+I1C3iB;U0LWdeg)cv$no~_htHreBnWcE4at=}D##GyXc-n_z6{3XAWEU!;I7X#L^V$#5mjy(IPp$GWD;e|oOv5N4;2WrZD z^d0oog@x`&Xb*n0;Oew$fy4Ps`5UQQJQd=APpPOblvPxGD_Y>fKnq~2TTqTNe>M|9 za<8n(>i@1P^~eC~0oN6_0q!_I+JGxYM`PpCzbmt}T|!V7bGwq>9K8&$tOI8aRG}BV zHXnZNr48NKW`#_Bc5~s+{QeRB(URFD0Y=7lct3mhj z1bWbnVm~)Uo*~IX<4oLx1}*ZCJ5OiLugRQ*@aacIv5(B7eRFm}2HxRB2tzao;q@ZI zN$rDI4*xqzYR+Yt7CAU@FWjEckH!Zu>6j)eFB=48l#gvbjbaBt=hchi%;lav((=$` z1?F^?mbpE0U_C<3bpdpghfc-5YPqEirTHL3-5xLlojWRhJqyIoDf1Q-I#0a6^~|**V&F91mt@ zIrl2XwRb{ z_a+a&D#KLsrK~qVE{QifV?THgB=SEFDV(PKif#PwL*}5tsk&c_7#+WRNo`8@c3$FZ zX&ud?=89*LPdVF7ckB{+f3QPCQv*&1+ z?{xtcZ+aHw@&-xH^h2$yZk+1*7n|YP@0J*Y$hxudSb{F{~>)CODowuaQ)@Pen zd)pIH6}>9i*<4lEv))#2 z?+1sNIpORsE4ORJPsxUDrm|K3q*yzvS!;B`m6C2gPm0&%F^5aINDW<;VR^aeP0c~B zi@%H?6Sy4a0qRynd2Ac|B6W6UqgC8L%0jOcMAtJla zbB!`K3Y|H#{|= zjcpwU-K0~#%Beo7^O=p|NY!N7L^s%=%6;a`)St0b6vMtb2P4m(br-g6BFTsb1$%@q zQPRoK`%8pxcPyjYF3w6{Jq4Z&@EqlnmbruJ0A|yaR zV3HnDujDp3D&yInAEJq{Ht)8EX}Qh_kN!uQA374B{bU6zv%ZZ%R%;R>BGz4L6SqaU zfGAJPF_);@ziXawF52o{wF1)fs2RHr@4mt0|6H({egouVZh7{pDq0O`$7tY{0EW~W z>>SO~u|w(szs+kXN@`#AyWp?HZMnMe@xa-P+Et;ApRxQ;-}V3igu62`m~EDpfKq7a zf;bCGm3~`CS?Ms(%1F_S59aLUpg4S14N}I1unr6dbp2*$0~k}Vo?vfU4+bRweyjIuL7f`9H*G0=uE%}Vl2Vl2?9v73VI8UCaT8+ z?||A*E&w^`^B@aS6{ty;FV`~$QcrCg&at5GMi<)k5Y2nI?@{*UZ<~LhO+=>X=s5+) zDMU=Mk*~urjWB zUVjMvmO2=Bji}Am$opci4#3M>FV)L4D4nvajK9UJ`tI6j1dd?UIznHj^RS>6PBc5@ z6GHF*p~Pw(G%IRp^$f_6aOBG12jLNg#+0ct4tsHd{;k$`RU6+2;;*V+Mktq^7g1zx zI}O)*<6QG0TZKllirKsW3*9D6i-rjw)!CKNSX)#VP#7a{;}TtdFA~0hWSF>&R|y%Z zveo1g;X(tVt+}(D`%iHSxSF3@B|O#Kce+v(^Hrp5k77Wh&$szX+6kIzb!>tx!3|W1MWoKAG#Ayi z$%%>&uefUP?0^1;J@CW-M2%DrF)CoAM}v-swl*{y;_x&rC`Y!d8JW z8A?D)2<#yzJCwaz*1fN93>)Qb47{1p=%bAG=6&fsXU9%YmXPMMkG6U#WhzDSZl%;1 zMEw=(xPurYS!d;odw_;WFLi=5dJ4KqjUz#PRl!8f(ASKYu! z(kX>Ex3KPMK_0{T4vOvi*@@unhZvC_%|%or%vJ@A=Z=$83c~}CJ`#2e)fqZ)f=z*w zlzoNXZqoPQxv0HBud4$B46eRgGFm9IKNs-iB)3fld-&+7O+evN9SyPIrCMD^j-^^i zkKAEB!YZys;l1Z@Ofyym+xo4;2r6NEVb8$R=ZAKl5corE*Z(bm1F)TR94Eq)IyXwq zqyje1B^RZ@P_CIkOav?9_t20R9ZOFWtYkhUZ_c46$5tV74mOUMTw5ehCz>c74_Kw+ zM!cud18x9;)%;l7OAwBsQo;fLY`K5-U@)S{AQ<;l?(7Z1SvnrC+p)@WcP6vW?|c&A z*-d1jyZxokl1xK_lgEiNmAmZ4*Gyvcd`?@q#?1YmP@VA_s#!N#ZNO4vbp8ha`67N9 zq`t|sG+`7iVZQadl&SuiimzN~Sk>#yAm}es;1$?@&5nbUOf^A+6QiuK5zo~`B$w8}@)~P;#4V?~k4MZMfrh<03w*GUn&e4kl z_g9fFK1a+Vl*q0wimXU^#q}Zc$0>e2oWHs|TN#!`lO(JMUX2R_Cwa@ummHW_AO@}$ zrhmC_;Q3njoa?xA#O_t(aPhqddS{I>g%ITSHG|}PJ+q@x03;l`tsYM4$TE?Y-g2Md z+DVzE+B+cL;;9QbB6i-)H`_|OT}dPZ&4{qCl}d(v>R-UqPz|4W42u#Vfipb?UhUkSjh3uQfuT4#ku2eN9cE&~Ds`FOV8}3YoWBKf1Bj;C& z=w2Oc$%4u^lk<(JF=IXr^i)^|5W#G32aFdnAUnekcTW!0!xko6b^Mf8$mn|G8J-Ra zGUd9G!y}XPLnvku&XKTz2Mnam9kZ5qv6n9Ne4@H~udsD`|B9u#_3w1w^~Fp*v=_Tv`0`mmdTGn*2kkmcLz1qM=Bh@+Cls}F?} zE^uy#nWwjJ7bShY)C|h$0}9em5HA!mJnv2=(w!+m7G?NJ{vc>G>Csf0*sxe=-q2@~ zm}(WcIwnCtpisv5FwTwY=`eaEJ=ZQ8pR*W-fl%)ow#16sBC(uW2FPDn2HpGWAPpF^=Ong;jV8!>;d1b%?1(I?mwnc zZ$l+OIfJO~QB6=@?|cGPEq$uWX(QX_i0-=t#LJJn<$lk;VSp5~ho>_5dCgA>B5Ty! zKGbbb)HXAu(&72ulbn+SwTAxKDl_RpwF_HGW@hA~SZ;x}_+(B>{sZ6=j4a|xEtKZ% zugd0Ge5M>eqc>wRAD5q2EALn5oSRBHyoajl63`qV8BO)gc`w%}4}A^8I5L9^66Efs zTHQH@^4^v~vsZ^csgjN$QBxa^&mSSK#3&?Q9bSbp*+~ z4S({+);SwU(fs!%1b@C63#z%%Np2*_QgvcJi5#SvR`oOWeC!(jqLH^i5ciz{^}8d{ zS+9V3232b{hp7oszTXo3rf^QU4VvEm1+k%!N?^VK?e9w?Ulao;uf@)o`|viA6*;XALAdo=96kQfxACetp%Bmt>B` zE9Y&1ih{{OE;a66-3A<2xk%Gdd)_;Gl&RT-8o?9UlFu#Clg|j=MeaG(>0dkbg%_7Z zeSw~T!hD+a+t@1-J9NqsG9g84=nEWG4&b13342vuRW5r@|A z!3^?Zc6_?I8~=t`UoIxKd6eTAG^;RQvO~4fOuRUdIx*a+&#b7m8D0Etf{DS%#$)_c Ph1780b;q3fBi$^D!#LMH literal 0 HcmV?d00001 diff --git a/secrets/config/gesis-stage.yaml b/secrets/config/gesis-stage.yaml index bad5770e2a7d7760954407ff1a56e60786d4df04..7e57959060eba2c06b48d1e4b939b8285485b18a 100644 GIT binary patch literal 11420 zcmV;NEMwCEM@dveQdv+`0KZzUofUL{E~1JGVY0sTyt{BkNi4OzcS>6Tb2nFk+o(T{ zd8<)_5*Tyrf%hHA;x=}Y#I%FP-s$wTEhyfVw(e2`UvJm|r~q_bRwai!+abE4wfmNU zb@o|lS!Hu&`Xbmja~ps|mACTeGi{9}!tUQK5G&4b#M@OF&DyFGuZMC|#aJcDBx^|d zJF4d%U;9mwb$8q0>jLRsn7RZM-tJO!pIwQK`Hba4e${i%gDcfDRLJDU{HUvQe3Rog zjM77&noJCaQZF)*R^3v~VgKATfC5lD1lWKx{SaulLM62rdoGg)gOG~wWkoBluv5`b zQ|K+T5-M+A>*co-eeUkCan@vl7Hk+5XdG0i9?)N%K*PCq9(?)!`=#%K$kxm(VrRGu zJC!Y3vVG7Ubjm;tPIr+S0UqWd!^Yec&^Y%HD7}vfM=?_KX5dSn8(C^@pztou z&ZYmE&%E&=jgCmL87z*a1k{#>-iw{ze%Zy0Gq0PD)PhRus{&ZTMV`o!m6-(_fV*Nn zJ9b)RI$2H1*v8sU^xL}R)(;afAAL`Z_s~}5An-O)r5kB7!*+3dq7)x|VyZ__pX(0i z`LmV8OyArz%21`^AqmZ8!h=&EWI zDtgG!T*xJO1Ou)h6UtAYL@v$fIR^j>T8t_&2K zJ*sk&zyrp)f|DES=Fy!SMt*BjryN@{E#-B4{&5aZbimPZWPyL3g>9UQ0WI`Ft%>#9}~Hl8u=Wz!PcM_0y|xC5z32OyuLk>}U|C7NXNZGy>sPhrG=H44LB)#)$cu04Jo{QA0c>_^{AZ;CxMoer;;VNZi> zxG>=Yw0b)kxtv_X!0O@i@cHeG9GH=j71^4~fi6{YUrXwilT7Uo@O8JCqo z`jkE2FwH+~wE66faMbs_D81f-Dp2X!ZYcU>c9tb8SL0ny0$#poMaGq0m@}Rxz9u~m-{^ZL zP$mjj9e>)H{-5{*Lh5@Ogq9|M&?W86-Ekbq_vl#+(xNH~- zGlyRK8IPa#A938xGocr;DTlXdesSV^Amgv_+-zW)#HY|DM(%7Kh zbB`^w4|$fx_XV)}{8{;5=!?$+aqc1sOJXy2Je6h)-Rg#4VsZe@MheU&3ZRG;W&;Z# zi5e)6W=Jpi{>ndTsH|OHn`N3yOc@y1R4WdZ@7`RJNi%QmhFHgwgCyeoXq7>4$B*^s zFUdv`M9p*U71S?i=Am|-EdTeky^a0d#8ReyG>-G8O|)l2pHSz+YhUHtFK9gzwh1Lu zZcyK%{X%fC2{<3~JqS2m6&8nyTr_P(Bsk4LOFK`KLvs)ndbrl41G~xiQ?g2nerFX> z8+Ai~UQIe=hCeL^>3%bhPC%Qt-U{VJM{HGkiP$%s#T4QUHZ`c%XY@J%Eys4mQ8&pz?FVTU3pkM31l!M zu|~60k&)LO2(t081CYBBbB>fq^r|2()VO60W~dwT!G+C-s@!ASf$)y_MWKf>D?*Pb zqztrlePj#&uzjD%?1iO|wGmakXAJ?;UtE`5w!amWmAC_^KwV)*hs9gy`Gz5-IGa`t zM{>m+EF9CrlV>3t`)Y5jHs9RgOYu^75WiI(S&2$y-z4(7a!?;Rx+W)7qC>+rRxdRd z62wHuqr^|Ld1sVOMR-6tE~lON7UM>IeUJPp89K}IO1m81_VnGOn!wyzDXLpwZ7}h> zen9N?s+<}3f%7mjO3orrI7WuBiAxNz_BEprSHN~3v7Z|RABZ>RcRt>j;Pt$LCFige zy@rktgmgp#K}!F~5-i+KtewXyVblW=tA@8bbG5mwsfU~@d#u!wB<}QOqoG1&rRrlU z%ZcHTpvhd?!uT{W|E5@g_fQq|z(5s{a^Yk|?)cIDw=LJMY!nfD&>;hKOwP|`A%cm; z!vc7wyel50DfuaBSRb7SO!khl5%_ybyr}-g+8|hPbp4P6R=s}r`r#o&tSL+*Ixu79 z-%F^|n$&)Ho9>m~F-A{1tafatI+5#V;hlrKJh~i8e9w4 zd8$~p0e7l?{m*0~LmBhDljn{ZMY|kmrtG*+?s%R=m&$)+yu-s!-)c`m^;OMl1s*^+ z=RzQNqaTv2<6=#i9gl7??QJzu;;H(dc|{57932Uk-<#BK^%e6Y5`l-L$mmlyU~B>g zNmIzBxQVp}c)bq&oJ@qm+cl;rUCP<_*&~bI$&Ld&|9Qs*H*cK1t4^<9Gtu%j;h+iH z+iF)jS}t$-ShLBiYKK^xIFWca$L__@d|ny&@mR19f-0xS)qn{J86s!9U$D<1^?Yv3 zOn_qjYFpIS*8yH+Mss#&Bk4*njg^_1SnlY^c#TyDg6I)NN7S?3yg+^f(Q~Ft^hPa# z{*X9?w$Zu&ta0eLKrhIPcbG0x#|9vF7{m%t(U18MLJYZpMf1Lz3Uc4bWu66jrpa zFe{&PzF=8rW-sii8hLVJfrq`grd=g@7 z*yLCRns1vUsfw^Q;77m?bYbr>XTiXo;k3RYjKVxKB?nP3n}NPxktET9#z~kNt9n`1gp0nPT5FeHWhbxv*ZMe1p^XCzZGH%#7bXctyt8znB97R6< zZVyH`*?GUQd&c0#-Gw~6;oZrg;a(WMVS28Wa(4>40v*#Qb<|gVfIU;8aShBF7LJFT zS@D^uWZ{p4l|w=YB6@XjE=CR%xB6ngBLK2*ChrObm(!+72V znWZR_g&Z_FGOGOc=|d*>0%U_|S5?&9d_~&()ANht?_OyYRXx(k-cuvbDzgZDQt^l( z)(0c0F6MqKHOo`|%T(!K-(Z>0Uk0H}F2eWxuxf0WZUXd7*tz&Upki1vyM&aS5NfS2 zU*?Nf`Z;~gIJJE8zn z1t7s*%jimucjF;w8_Xc~>%J;$k)8)XJ52YQS#4lbGJKwX+E-1&U^QjwPRRLtW@<@o zNd>QUYi#=)a+3khG2{+$oUkGU$yJa#jvw9PtQq!v6b`FBuV|L!K#J%?1Xc=om~IF- z*Jl>4!*|8}ttD&53}wx&RKvMXji>bMt!^Pw1erLdyV z3_*$CNT#emLnDm>4-)$F%fM9TJz@c^%PX0Y&3Eu9WZX&X(BkG-EVyWm|2@5sse?o} zSX@mz5*OzQ$kb+-gyGnGjD~*A?kxi)8CLd3NHUr(1=q|oae?J$`JQ?RZm>-4hJf7! zF;H{ktu=j;NBE;S#WI{_%1r&R1|V%mnyGo#&?AB4X}*%brsjhba1Lp#MqQ6oCnJib z0rbL?>c>Od0~|3ew1y$sH?Nng*_UD`YHxh+hUoM@jp+0C_JFu2M%N$$jEWKCx80`v zme5Q(#Ot(BA+Oi+2b_p~reznXqx0vw0TA|c3Cr4I>`BF7ozDpQ!_;lTzIIKqb17_jaezQ ze9DBAMUE&%=9Oz`w1}8PD5e36wRkt=GIF(_&m17n9t{+8gt6H0k&e}b97bWL&9=Mz z;OymkO?8AU#-n_x#j?77MI0wgiIal4CbFespiAe0nDfndJpWDb0~1pS*U~g)A>553+&}) z7)d8B5hiET^=#`!6!L-Pf;{6**&5NQXH;9%k~J}?g218L@Df9E?&NH9uZGJqg)}y5 z4x%BRCnXtt)9dS#=&fpGoomnjuWk}+$%V&(odebWUZP8aG~I^|5$<4&=Je3ETamQ zHfMV2T1nY{|0qUu;P$va>k+bJ5|xV*3F)nfbuvEvnak4JhmZlVX-Wjw8PRpm_!deu zd8{2IhG6d3P=3)FnlkPk5%XMyq4D%{b?`mzr$fg1%C(`*7V`?X8kI%OH|{^MUEiof zr5CkWG&}ot9+D>Av5{wa?*IkbyqXu?nGgUi8V^+=OiMJL4^O$bJA#3=r4*6cUDS6i<^dIb3zOoBZAZNfWYec3m=7rww1cMnI@9&6Z)w=F zOs7SYjJWjqD`dneY*zY8xN{jhg(JmgD z&n6_2(h4SyIvy_^7{T?D@P!;8v;xE{B;HZsGHTZDhXJ)jR|qSI%ly)gZ*oP3a3)-> z5MJvDiRq>p0H#c-+?ax(zvxfF}KZQY|Z&k|>m4MuOe_kR(juuubp>Ivxfh+$# zBZ4sWW;`$(H>Y#^lC4MC!KQ2}u~8%jDO3lXzFoOxp_$v-8To z10G%P1rW~{my-nfjNZxQG3#rG*>>(`jy*59K8kfJ2N0E{?E)obA6!0hr)zZs69_vX z2O21FMZsCC?GwHRm=;dCZ?{X(WPmqi@eT5G!2}Mbh;M5;xHS-^C_sqj??V z=hpEaC+XPoL;*k$GFtR)5MU>2QoD=9xJV1u1;&?1NUeacn15!Ph>VK7OjN$0JxbXy zjp;aq?*-A7C~`CkLzE5eE>Htr#fjz@Fuwzf5Z7Yio`^J+ssX}9pNw0`h3`*1<1$mb z^P#;o7&g~MDb|mK#*@VNkz+cltHWxWkh1x@3n?rJFlYKcd>S$td5+b78fJT5X@U3i*PuIO3{8)X{I6W2yzi|hU6SR$ZFx}k)(O1zF^4F1EE}s z<|wWTtO&I7TT@TmKA8Ix5OOCcTPT?Z#E$_I~GDd{L6;Fb3BJlr-l9*RG<5|;f(;lYU|S!LAv?d&TTiJIW9;` zN#A;kGa_#XqfoER9Hi;A58?M)Ih{ z3zt)*s~k1dVqWQjd{OhBN@6&-DoyJ>au5hcghq3EP;oz>oiV6l$%e%EarpPIiLsl; zWpT)P=9gmbh`FJ?&9hBq3lL7avyz#g&3gkrjlZiP(_FdH=lV>@P8k42Ca!RY6Ua`W z$8lg?Jw3panlM>V7&l1D1h)K(0vSJ%koPc8R%P&uQk2VL8yUiZQ~8k+ajD$v)~Hp4 zr^l>Dnh^gTeu$iSDu_vXpDApVVz-P+7FG<8{t{aD=O1CWL7PR!mfdJ-TJkImuZ2L= zD(OQgAl-S;wc|fW1M8qsFj`S-K)O+A@kf+vQVe&YvS&z~W{*ou>E#v@;Y?|jjQu=4 zC*`DV6!YKa7{c5SbK^}Q15Q@>q4ESQyoQWzd+UKbNJc|8S_)}|VlQQb-|uXU`A^N< zsH@TqXlk%xf}OR^cGk@=$$3gApvxC9G@)Guoe2-D;*TdXz7fESD57zQvz4otb|7omipSo*Oh>4e>x%OWCw%`z~7yEi=FV_&xNBDxIt9ayA#Daun8 zCgn)->0Dq9(!=&7Qmq>ueHQq*d66dO6mMk|%al|~D1|fTh6Iu{5~)k%l@{3}V?bJC z0>V5kNA4i()pF+^IT-$vS`1$(qN^o_aiZ7OukY^aKsfnOEjK`jaj-^WX+{NOA#wb9 zK!zkGj750+4yfWbKZ@DD=bwli2+H|n-V75=7GMa&+(;IGg@5>R*l zZezFP@+Fw1_v&U51U7=b<5Sr&QS>mXe}LVkLNPi#DQw%f(oENCk`W0n*yQS5*76oa z>MVYgISalkn6qE`_fZi|!4Y#&1mX&MsgK!5CZG1nLW#$=R;S<}@NonenB@&9W&oY- zm?N|J(HSQ9HWR5K9Dn>pGhVe9e-BCV+?cO|Qtl;n850^A$**q{DXn$|l;1c>}yS=m}tJyH# z;Xrbd_)nx=nar6D=v^U{m~{FlUo zwN}ls(GW-}rB+N=NgbQ+O%(#y28Md5D$UX;%i=awh(g1jl{>`3a4*4tsg~!ZJUUfZ z)9Z*r$H+X;*Q?V#d4(OKd;dd- z&NuHgjoRxz`Zd?r=|4U7=k!#rpcu8CI4=M>iNrq4)DVADTF5|a9sTx>90AGD^CjgY zj^V}|0o3Z3YCvjesZK;?QUsp$YLznq6Q36jKGYN^5zjZ{JKHHO4D<|Ikm=F;3N!ng zQo}iT8yCrqm@0R~CIUw`_?la zc7+V2DMN7xv#e?=qPOdhGsJ@CaOHyWv?ze40$>VEV*sTZKB$O=wW2Y~n?~vke&bCy zmHzdHk=_MMW8D@Od!6z~OCBIFmQkdXRn^qm#=u>s$5tKWoYht&P;X3m(Pxb0)}dH7 zvh>YD^-wYcgsL$grEqW_j@wth#FL;7QoRJi}d`*mKmTJGr0nm+(;hkaXH*+KKOr(tg z#{)>kN|DG{214o3@a~KeR+a^X$Gr=aMC7Rey74A#B-5z6|4_32Z%3p?4u;j55i&uPZ$4DBxe&0>t)32l6%4m{Njjw6=5t7hlm@OF<9< zRTN&!%3PA6;^atse*?EB9BBCMf_2fPo;AKN5p6OKR8iijutm66D#PjSs zJ*S@$Kw<25J8Dtleu*kG-wYWV?}VLiV2)9VOsE$%g?VJ}#v@CKVUU}&ztkhvTzV#N z8e60pw}UqtGvw9JOsgOnVh?z43M$;QF3ciKbUsZ;*<|*JA%=CVRvjUvC#0{^4WiS& z-`^U~*HbFRc$HXKO$~VE$Ee&UL}1+oJiLy3QJSI~*g_-pu^?)&uG`%>BL`3&JFwTd zQ(Fs$N{9TB6vE__?l@L7RLs+l4s)03wFJ;clj2q0$X25re2Hx1s*JvT#8JQM%k`XX zhX^A=F2%r4mBtN?Q_;*YH&%*lmbvyP(-Dy-Fuq&C;TBvZmY1)I6UD}GT=Q|CYfU^HA; zj@5dFSvN4G3s;HnEv_r@2RaUwRXSVDA42$WK}s~y>pY3qYC$N|DT?z%YHwDnGUT1$ zRbSp~%gHQVnJK?-Sxx+{ISFhcP6 zO>DU~V3tb%>N)`jtr)f~QAcgVFgm#KQBl^5bKI(>si09ar=_tr<4C zxz0C;d53fMH{_{HyMm^M!E7hpHnmz$qq*EA0q6yk3j%K{fe;uOswWYKk&psmnVM1_ z(MQ5D<0tE{yw(iFIw<|jWP*q4@zBL_6-8Us=WtfL2dZl+o&bJo+u4SbcuqvKFfF2s zs5=1KR0|34>5)r#w}J95RX9;s1x%s47&N8=utu!_gOiKbrJ`7^r8HYLc|YV@4Rfcs z|J@(i8#1yV!Dh!1&T)JsVk4IiUu8tooZVtu_JmRwaSTmWMxxyX)kZPwi_+;%KIT$L zJygSK^M3>EVttsGH-xDAf4;TA&K&q^f%7L&8l@!H^7kYn5y_0cCU`*3dO>r$s_jVC zOtjluY&tOgwD{Vula^^$-*N&JI4Wcrd)LpVe6VZ#MOaeJ8^>-)hLMUNe0lI#ppyn8 zTs>k^Fv`RkY{-H*rBvMq5lo_T*(W~bfOQ@@=r>oq!$T&62{XZxDNQ)6hYg-x3wBlX zrt_z3EMdqnnMeuX6GmKwpMALJ%FPkN&$PGRSSFm=))zoEs8YLDp7WT@-mu4gda37Q zmTk6;KCD4>6PFXr9`6-h%gaFRl08d_3R(POs85S&p@@J;eOysD3Jw4RFtCAx)fv@> zy#q?4s;fq|!HplwNrbIH$&OWwfbbc&fN3#|jaNzn^jK{wXG39}oFxnfv+ z&)>Q96vefUf|?fe0TYI4YXKT8wyUZHOq6}r69=UNcbzDxFIAIAY}vcpK1n08+|cqQ zZ&Nb0fgzS%91U|Mg*^c|HhZR-lh!s?Jj+ED#lJ&R2sK1?xpa=O#t__!TfuoZX;VAj zf~9a=&RzMvWcQaUm2Or!?MubF;`)M=l-sv@W5!F#BWj%;bq}wo4Ej~^`Og$wPEKP`LQde(Krd%Im?89} zKB0?f4aY5PDCceLZT?QPTjP|3#C6E8rPDi|e>5lubkdZpo_=Iy!kuV3^;TC6)#tu& zx^y5lq<Q^;oip`>V^(ED4hO3z z{bqZ@>rpD^io^pPRJP3{ayu})Nl#!Rka{;IzwCmBS^k17#XfLB#tm8+x|la&m=pN1&7oa;F{czyj_u2YbZ<_ilf{I_8# zs~}qFR})m0L#B?X3fs4cnjnhnJ~A*Bmo{fGFOb#UvhZsiGSWYjRyg3b_4dnu8x_Ch zkO>sBl5wLDe@X_Gpe{!*A0x9Pc+kA9|#EY4JMWA68$|ow}?Spj++-dW`T~ZW!r*HW- z%%T-XI$_o7o){)lL?|#|{XDxBS>MZdTMQB(bG(12le+I^8u3&|03oix!xtP73%O$Ad3`+f$ss)d}cVQY4){ESy_<4RTxWso77SziJ!ehJvG8JE;i<>9%Mg8mhD z?rHB-tGMScpiI)dSymb}HO|+|R8#L+eWB&?k?zOKivulj9tK!kBIXTPS>?89rGA9I zqH;nyrY@)9<+XrAsSK=Tm*5@W_h${)K{5@z|H50MW=bqNdxC~e4X^iw`l#t?7jg`H zil$j^E?hdOo@)L%s2M0j6IbX}^38a3>`z*IrloI?iBqjYP<|chJIwaN@FC&4J9xF+ z6Off1$u;SQVs%%!D7ze?gd=*w>C^HY3|O`CR1OICX3Cxz5FHZ?Dba`Fi=rpumJv=d zx&kxC=c8dK2pM;GV^-Ln31Jql-#|NPsKSBP;`bWKbvpc5{5`OdG|B-orlu%51HFET zUzMbTIsY)b!0C0n@ODgLsq!=F0Iawr0Z+UO_MFci6#oL@uqv^Y2>!r9@mhcxa`2mp z+{VcwS6MSri#>PZwWTg7xh(gm`$?G+#3R}dCk&Lzt~Bkzi!V&HW4d}_0`Ia(#@{m1 zGM>FG0VIpuN{aB7+Mp)p7+Ds&60L+))n>l5!Zkq*z=MP?H}5=j-JE|8%$~ZP$wjyYa~G7ecODku1Cc5G;eUGYUU&YQC%dra zcIVJc`k+^zeEf-9^whfxgb&Jm2?6s&&B0#FSChC3L7Xca?_>Z2fde29LfHYt{zBL)&$^RQ6L$-&cVJ$QcBUxL-;OO z1AjL`-T-wl&kp`2uqTagY-LPX5vi9Z#S(aFB0g*!rbZ6m6z*=T8xDT;kH+zAWq+5* zjsIagx$dXvftWu5ZcSD1{i+acD&Qlihre?a3<`f+{(vE=#%V=HMizZoLg7^jQk&el zT_}SwdIH(uOu?}Bg2;@}+j>Ua(;Bx`AWV)b>}ou=CB{IM4wERyjdBd!U#qrLAtoS= z$q1|h7@uh`%JG@7pK*JD(?G`b=qNOyhIRET=Mu8LlkviOdFpAi#~k22j|Qhu!b$NX z!Q&P6W*AE71ByeWzRnbQ@4qo+=+IJ`8uea&9y_p<_Hn%z#<;%;rd_2nX)q_uU->0S z6EX|>Y~5KGL_)FlmQRm8hW?_ilx&3J1$57GTtC0&ivqq#%mD#U5eJUW<9abWj=aj= zh?QiC223my{?|KA`yS)#FK%S}HQ^qkD@EppoFPpw)#k`%jc`}lYyVwFCC_^{VY781 zy*6*Q#Ijj@bWIx!BJ;mnGgoz4Hs@dxeUPpS9CSYnoo%Vr*(DZ2bCIN5* z&-#%e=R?>gCwFzNp`n>VV`nRNIQp#D6V*z3#N1N0C zBAR0z57^k1&VkiHNUDF@7I=DDt5k&0#jI#Q)T?l#u717VRDPl9XawrwnlHqkQ;8AR z7o<1XgHmW`Il$-b7lRbIUpg^jrsI~Qo@Pb@AHfbJBl~{40JZH12Uh530SfUy?aFSu z8T{ZbDD7Llp9t~* literal 10989 zcmVU{AqbyY2dUU`W1fpcBQ#REY$paRa3Y0D>ZZdS8rDZ`Q zvT7hTbo<+ZRt$qa@I^W?21LL%&9aia*j?L45u3>Kn6R*7sBAQWp!SuUq4xjl;l8L% zshD9b*W}laF6$&uuv*QJ?w?}@w&V39 zmO;)kDu(}kdrNS4K>+wAT8|hIOb`%kc&6(iiZGonowxq|EP{A(LT>nq~LP6a&(G_Db z#RR)al(C!Gl^Vwo?RqcdayDMQ496LaH_OqMGhi}roB2!cl>(^x`u`mo=u{>e`H+lFP6><-$z_w^?p*l;M z0G4VIa2noUVm$DdO6qDjclCs34pGMFrgxdi(di6W35PNi-AR91(i_ax0d{9e0u^%j zJiRKlj2lv>jlCv=>ph4*aGC1extA1hjWP$?&wBOs&85uGJ5itd@d!_WT{S0^5{k9tkOLQ)=7wE)_ylN}yY~%O{dT(m z!Fy}rfk30gG7&NCg;@OTV#-I)nV=Ke<3GfUAaQe8=>R(j#K#S~oj6O2z2H#$?m{X) z=)X+y5$)4%-q-*|%omyN%8?>38`M=uI%Bkq_Rl}16}7N6XqD_4!m?C}vxZTk@RLY6 zv;@g1Q(qRF#(%WYQqtMQZt<)`R{LtBDC4S>7|;A|5Dq%A_)~RDykAmTid~@8D(OAa z6GBHt=sjL3r{vdA4x*lkk)abNesDP)$*iK}(k|0NkAvOv+S z*TUF)+mv66eK`$QV}r+C=FX~ha#k1)=W)QP zg+M@QR^LhOXVNf$6vkGePF{gy3WBdpf$DTBj@D%R8B93*2P01FC zp5dhwshgVfV8SZ#><5BH&hA?>&OTyG=J!FBF8>RpNDgYCQ1wOimyKNix*2Y7Rb*Fn zFBs4GZ>CXiikFx(&p(lMq!!l9rf*verKWXkW?^Vt5+Vb;#DTF!JDXGJ z5-V8;xmy{xM)M(`8py0L|J03J>PV){BDy1HjSf;CxZD2>4c_o}w?}=b(OM zd4j<2`pW^Qg+?-*=C5=T-J4m-`tzvLjEB91O1;U?8OXu^AEh2&v`HpjVA0My;f04s ztmWFZB``(zZ9z&q78o@;%eQttU-qH_sZQyl@gsU-mkBY|qvVmjyKUvVZp=Kb>wt2P zBPq#(AEVk>nlcFGLEwCXHPwC#I*vUI3v-EcbOblCjcg(zG}!eKy-ekdZ`=ko1SjDDuOYPw5gGr#xwxf}>~Yf;VIu zy3Gz+XZ09?nXVV`P@m_^73^;GOWa5khz<|*wYL%fMI~b3qQ84;n^x8OqmjAhNl%zc z@{*-(%g|gI zJp4{`Iv*QH;QLv8BXOEB;RrL}AoPus#R&VRQxVRYsTr*F)nV~Qx|7>2A-+rsM0iQ0 z&0NC7zlFTRBkkWx&mE>f`RRPia0juoY&Ft$LpsOuir#yMw3yTcubnbquX|J8(GKc$ z5tH;2vJ9<1<6DjWG4&=2M_84c(srvRKPmj(HHYXQP3d48-G*GNj+R0YKkW}0oS;76 zc)=sd=w{B(b(sIEuy$pl@POFM(z;c`2VL`m-?}-Dg}oTzt))8kG1 za%Zr>K>ZJ7iu{_y1>g#2k1+-JeNE}I)JPf!8m%->HMhFtH3KIdkuk@`H9v?s+IEw^ z4aW(b?xRqgau2D?3Iit(IZ~N2M!(l}&a+sVhb(?mEv|98K10}<5FZC!@3}`)ZPS}x zamktiHsUcYo>Ha5KcG(IVUP5#74)1Q6Au=)Wq?)F|uHA|l02 zr0KQ-GfjEmE))xAK04wVV@Lj(nm6ng*KyCWFCRa3>$7zFZ_c+b*UwMCS^zeu#D^(A z?(`g%#`mKHd@?}wwR+3S4z@7{OqkR-=`^YQvl`L5Ps<^VBRUQ0VR)z7k+uo?pJr?-# zdOX24=GJs*^6#dX>@Pe*Y&Ud0ja=wZUL@(?*ERKesFEh%C zILhtBGzg_b$I{#L*Xw9i;>Y!sbjC#?YFz~ie95}&!6~R zjW44;@);cBx+HQcydQNK<)>%^Nt<@Y_M&jNCCxBMDPtKtZ*^A*T3z9a`Q#pi3QQCW z?O956U`F|`3irFmhPZYpO^uXy;qop_j7+X^)hN0;1e$ZW?xUlDn#wdo!MInw1<}Cg z{kcC6*h>EZzSRrA7cRzaZ1_6;Xh0~Vj0d|Tl z_wYo8<4ASODstleHZw~ntM*$Q;t0{_Tyl^V)I5#WW16zPpO#M$u`r=$;BOjn&+|jH z#Er}zU@6Q>bG0Q?7OFQ6SDclw-v708WJi)y!u;zLqF1L$CWbOb<7~9_`vKnvrB3LW z?q9^G42;KgKWwt=hREg~Zs50E&ItB1gir!FBxCDsY z*SIe5(nF8Asi;2SEECY~V#;nOcL{NFt{ve~b2y;?I#{b*+dt~H!BO)aV8})|38?PW z7-cFQYhORmD<`#MJe&RmhMgiuBgAuhb#wCdE%zz+T^nsJUA7B+$Tg@QxO^l9hY89H zTUE(9_o-nK6*@ta-tI@wU5eB$$U_vdv{mt+jlB{4+axREH$0oLVOru-jF&IZ7ZVyb zgmr;z9U@apI4&tk=H-J7P5i>D4=DU2aBxAC3bbDN0tI6}%(|LJ`Ir6(azT55ZH9;$ zDa<1u4L{zlX|~XGEW$v=jXg!A&duwHc(>o!hu{kvtH$G(>|rs_U|{5>*8%c&1kUtov#H~gzR+gOi<5-B{i`d!OZTC>0 zpWNf^STbB(zAv^~4Z$FkMu*a?6CB3j`(=f&=Cbl&;>!0mEU4Pq3o$;baMP~R0R$6t zwav#43T;6Vgxxl*G9kVknWA>y+H1tjfo1xs9=ppw53ic71If>kPGr_td64fg#9!4t z3gQCR8F$_KZeWR zdT`(*xUEfZFp4uB*lrVE5Ug21yq;IFKo+s9;%|V6NNPE09fb1U-;BRKU?%=jv1bcW zC1n=-apsSkc1G)!k?ef)USX#EpqqzsKp(0ek)1}f_S^S}91)Sjs93oAZ zwplhx6vPtNhCcinW%PQM&+T(PuYF8%ZJCRvFj9%y9oZfo z^9!$I7<%BLAjWUneDsi#=VOTcw1$ssSJ%AJNd#L~7(1Xm>#`j<*ZWUX9DW;CG+Bhd z9GPcxn%qu0mvfBpLQ=*|dGz6vpOTN-xMt}X+3S%r>0~!yR_IVjycvG%N^rx%Ck$AW z&expI07|TjUY=2VLUunfq(!E|@X$wJA*+CLnn@c}!nhweqZWpDg)|)ICeffzWN?o;zNpt(6kj-od?7-%8-;P z#JxHphy^bZbE^ZQi|vzs5wS~wT}6#QIIv#WUSN=ia`NTK-7pEYIOSs05$)$Mgb8{n z^^i*nNxBr^02ww;--rxP4#H+(?QeX1Ib0l);0lXsc*m|ZPpLqz37BCz91mchF-h8v zrMgSta2J^wwXbm}B0^yQo{B^stA-!aC?ukksc7U33-3Q&hnlTMh|;kM$IjoUSlYt4 zqwI2GcW?u2pLjHTQPTM5p|Kq+-0{GJj0E>gr={U4rK}Km=Rez z#UQ~6B^X}uCi~yJ)R2)d2egow6$k@4l`ptB)hU3fS3Y+r&*-6P@CciU=4ivFJf4j& zL7-*;q%p$h%28^uRKEr^9sFW!gOo;ypsj1cuzn6aG|1EUDVRRM~zl_UP)q+ZCv>?=%vXuL=c+}@AX7wLneh!JQep3A#j)ziH zYFcLP_JlZyaGNS@e#UzA+mzS^U|^Qpz#5E3Pk*F7Q?WncwFdoJtkht~unoB9D-f<@ zcE!Xh-x;UMJ@F68Kk(t=Md3Y$=2k11#%^-O6(KKJg2eOBB3yS@cPdSDX4{rJZUY~( zcy=g+Le7D-$AOj-#oB^HkhYGx^ib9Td6!0A@9k1D;GDR(h5(De4-nX@&5#ik8ivo; zFLt|B8_S~DDwrJq&nsw-!t4zrNik85<;`kE!MGt zS?P&iafcguU<8sa51aJ*g}oDcCpo0v>8<$GY1{wv&ejguRm(Gr3@1yDO;oOCPYHJW zE9}~5nDd{7Mfl>r?`kHGTm@I!))MUHrdbEQ+G7XH@TVSE4z%xvZ&ptJUo8cVBSd0^ z%mFDi)j_c*El8rpq2My-C~SZ2r*~?!3uH2Suq~Tfgjw?JYN2ZJi|=%$2k^l;AZ5gxeF9f(L!2}IthabNR-CSnGt zJf9D+XR$i!J>2xh+N5CyaKw5`j!mUsOa^-qPU5VRo<4m9j!W_w5^7Rv6 z?Id=;Y@4wc6vzsAS@@7{2md>&H5ICMskmc-=D$;3?Ry*Go-=4D2XC_e>lpe@CvW*DRZirKWGfiuovJv2$HvpB$`1c+ketFw1LpUma3Oeecc`Ygii+j1 zDwum}Z`I%8V=R2SBW162wkmeoR77msS$g6afZu+$Q(V;ldS5K+_RpIrcxYZa2eVP9 zU;6ZoDv}1B25Wk}!s^1qqImMsu-YeEu~OQ@+qCpW?^$+J6W z=6~z9t|@%|dOhB`NK|&_=~hdoRHE$01nf)rvC56S&eUPm1=fV_jE7>g3<5UH1F8L=`;^Eb@iC`ncHvTPdOS3Rx zd5sbalEh&uKKhw)_??fy5uj2$g*-~Jso`Nf@@&PDc##N*{|VJ|GC=O9J&%)_3y#v6 zKw{Wz!L^{zcg_0m$glfIvIZC$m?b?DA8%2Tfj*jNrj63f1rQtyp;?7sRC(%*dR29C&=^awFrXvhz9K2cwi_~{9t{S1YVX?Ix z(cBiUUazMiYQ?4?O>&FBpY_4jQUI`m9{t47Q%kyb?yJRik<>;Jr^FC^SIBeQeo6Yu z);>hTWGngPWE*TpViO5|q0JSC5d9$VmuEf-ACOwpO7h@|rm>$SQyBOPTRi6AE4U5S z;0_)kdF@1NsKd%~IC9Pn7b{FLOHvF@JJq1v#R`}Su+<0I*^sAfuh%4F=?pA>rmZ`L zkOqmLCc6$YqHS572|Qw{z9W%NM$`}$8yAH_eT3*aoh4ZHA)yq35j42DqZsMn`S*&t zmBnbwXyarIg18+wNLp#DyXqiBfc#0HS4L_zq^ZMxty_^oGGLMHwT+<4y;(W6EL`E= z!EFylhKE65^e#@G%;Ndo^WRma&6LNq-LTn>**Emo3!nTKT3ML*hAPiD2Wg0lV}5jq z+mC((J4J%>4Ie+hfZ&B-Dd4SU62t7{z3Hf{WgI|=>kW2Lb8yXy3!1=ZsT_Ah4|*`D zKOm(#+@X#`=DJ_sYU{($u}jdMWb)Ek6voFWhtH zO{oJ>@*~zk@g@Xm3Z}bzk$h}B$mhs8V{Qd~+dF3!@jpX*H_FaOnVBKtJ29agG`E{i z_qS6^rCuFMi#gL6Cg^gZ9|tgHXrd3~Lk|SDoZEaLSe}?dYWHyaI3p>J4|3*7L`}??w%Fvtr*{QQ-p*gpfzS^Zl7^(x_gULml$-tm{`p?t)Tup zya7zBP3tf|HtMJY66J04@zHV1$v&FG-EG50hHsb$3rboceLIt5B1t#UL)KdRUqHYt zIw+g@R5)s&0#J{=)cP+$a!FfGktg)tI{hKTa~2#U^J!U3i|GLL z+|lFKK2D5?jAaL*iCO~=d(l+r8$Dez%>4LIvEo=I+KzlfHr7VqeNtZ8o|VE=z_+tEol*473NmcU=`+>RGrzvpBqf9 zuwqD87jBCMTn<>1dgqKENZ|u6Z&=1}L}3~ux&+ZBIgW)aj#YioCpk+f+vjpewfvM8 zM>}7jT&&FcG*qz9az)^3CsDx}5xavuJVp^J5{mXw8lZLuuvV<_ZoQCov3`zcxot8> zbt6Opw+*UjZR@_0Vjr6VKr(P}nCmZ%?bGV?$^0#tHt_b`KEs-8pn4Ie_}d3WHuqKT z@>ZxK+XnC)H#iO(gThb@QLf{%u|FfVZbN@kOX#;|4mal@m>4|(O|5))dRDiI-q;bC zbbo(L{<_|?-wA+ZheK{J=I-IUagVA#316o83EfL$>Z1U`jdPzEz5r8c7!Yi>_E*UM zASuYsEeTwu#w{qS|L*WPR{Oc+lYaTOjtMF=1@mTXEbo0v-q`Y!`c+IxV$VnWg@Fx# zcE)7Y*o54NNN_`#N4Wg@2@NY!&!IXPF>0=EC?2<>A@KG3xDoS9{lYJJToxDIu zjQNN32VzGn=)-fg>9$nl>gKlWW}swmJpm_xD?g}smzK|XN{ORE_Q2JaS&r-4;~shj zR|D;LEs4&X_xlf2?0DbYZr$WBWTkc#v%<3|EOO+~oE zs>a*z+MA{1h!?3&MOEBU)h@_ajX1Y(ZS@b0vQiNJz+t&4nJG%BM8_+etSsOx7oz`~ zOhqoYI!IS>q4#@XvNySy#ff-O=o;~7P+(4d-DoPbF&ppO=^Q_?yw?0Se+rRU! zC7J1$j~jh)t*sxG(H*eL;$OG2ro&!^QYqp^(X3*X_mL;_%~sUF_zl$H&_uiIlquH? z2j7TAg~?9mx|7q!oj^zmXF@&(bg!p>2O&}i9e*oVB~GwJNA-O{kqapC#vldP!}jgB z15PqgE|sm@YEO3tSwou7M59z!yw81twnF`f7OQPZ$cY6NFawCD%hs_*fp8vHmxtYh z=ma#?u@2#Pq=8OTgqHvM)6i(G3waR&om5b=Nwg&TA}_3zA&KxVDMjmBrB4v5LMqG^ zUV%*21?6%@GKhVtd`phydag))TXMYLEBOLx0_H|$u%T1I1GIgiqoq`3@!3Msdu$85 z(4K?7=vKHVpe`2|M!Z zTDMt(g3SI*&4au(_ulnwfrri?Q;-bYp{M0uX&tRb;iBZYV@*X?n_nscjLK&<*dCX9 zai@RKGQY82YaN0^eqB&OLY;9;6Nrm=S|%dKuk(>m9w1TzRgj+t-ak!GOrAOVx`OE> z7I35)bBY!G;oxof4)09`FQ40WALy+C7Z0IJB>qG3o z=Gu{*2B56<5oYXNauWf>u@G(r5pt+U^_+tu98cly2opViT$}pSiFDZnMSOeEtVdsP zGxINI$B8>ppBmPEpJ|c%lms!3%Z4C6M1(C#gD8K5n_%LlvKV- zdP&}X3C!7?3+aLFBFlVARvCH+W$H!|Ag0!%K34#!F@BL;hKT$bGfdny1#sK&Iqu%R zYS|6Ca6K($O+de3e9`8h5+s-ulFi`)Ax7F8rfk2`(Uy*WxL}2I|B6Z{jq9@RHw`L6 zv4*{}F?>;RD&COl2Z|8EL}T%zeLHE{)4Xl!m_Du*KChXnw(5uiFl}*)xFN#R>gw!$%}$~rD`32BK*GXai#ezTT*}3Ov2w+ka6?AtMxyq+&jSlc ziE@&Bi5LbQ+dbHWPP|0VFDppd1#;lx5t7btB9;qgrbLot!x0|H7LKoAcTKfLW})Z} ztuX2w+A`cQN}@a6vhI37-Fm3Gl%y95k>GbaixGiH4yB?1aCu#(N$Eee_V?)y%VSd& z8D{bUbY6Qh+GUs%v{lpSqO?P;O|F2CL6Kl|>%9#%`+`sq)52%JkbJ2fs9hHtC!e?( z#A`&DbD6k{_p>&L#g}OY#!+vXwpEw1!;V7K4-UHPFQKPb#Q0GIE1avHDyYFsAO{Ea zxlgosG?wkr$~_yR8Hnm?;T`!)2H}x-aNw`?UTmiC3YL%SLR@HLbG}ZRKPM@~C01Lx zyZ@RE@OCs-;%CdXWKLw${LS~7f@Oz#AkyjxBr;rrgYgh8x(U&xGFb~^6;@tkn{(~h zjwx#9b^&nk?{v0Du?Br#w%CbA@kYwY^tzdNc?wT8NULK`zw@l0y%OP9gd zta1aOlnq9zeV?}^wnsHz>`D$VxX+&8IvO@qw$hX00U!eY^8xqbtQi%r87Pw*SaEJo zG!k&@I|sWpPhpU+3PfeCTS1{h^N$0A5Fsa)Tiul^Uu4u-7Floc+UqHeg(!(eBgBe^a8aPQ5M%eF8@PjL}!6*F2z~Q-Iw%wRYLkVR{SPTWU=kE#*BQ zjf=W4Li_c{EQN`~&l%2-icPgUgLQwAvSlfZb?+>o*44oDXPUBiaXvM+6fW}=4jog7 z4#aE&1e+{G*X1SbL*tZjh5b~F%?(ULH|9(@wmfZobbx$O9=3N#(nxGLyF)|(Oo!W& z-NyTp0jAp6tXVErv%;K)`HbV3+->nsQo^E5JaKfa@=FWl-w4U@#JDUxrKJYTwoYz{ zPN~Lzc7gs4A4+^H_gvZPbkBN|OERKuozY{|(bf&Lz{7ea^URaqK7(LM?lbblX*(>I zqN)c8xGpZU(h`liZQhvyZ}`0RCRz(OWdw)EVKVMkq4FK!Fv$zI-U z_mHkr>}N;yE8yI6<`bRZBW!od1M&6~b!EO4S9P&TgErr-=r-RJqzBM;yo$?o?2aV!gQg&HGcp=EE1=wEe|HZ_`sVjevu&WXC!P!sW!dSgoIQ3wYtUB^>eTa|9@2fzpWm>7a$&?dMxZV{+7*Yu<oNs!c7Lx&Z-5R%3L z89H(fE4bHx}hlsl#Hv~f>->swxNHyZZ}p#&tGl~y$fFmWvOAv9&a zlE9N7NB*#vE8G+fOOlhnf#1*W6_jPSyYd1kShGaNq6@( z3PWk5KQHf{Tzzzn2Qr7qp2c^A(2inv9~cF+0evr#C^8ud4n__RPN4jh;yD|W5so~~ zYewpA*Y{)fejigN$dtdJRzkeOOsiAeR=W+DMi#%Sm5?z(RQ&vzfs6N(knogma8 z0Vp`d#q#!xN7#>wxsC%`e$btq0m+}Sg=yqtnI!)PP6^v&`Hg0#k&xz?sOmRrg4(+9 z`my56+Sz8tJHEw(1ME4&2{l#;uXpe58wYOEr%mGDo6opgqGAuEw~=BC+f93KUS|L) z0iFbWWjV&(CS;1c^yc_4sHsr+3sL|xOUv6oAGAG6j9i*zq#>}k~7Y0qTF3l zvy~5Ab)}7v=(V-%dBi}SL{jU4?>11D3Iq@tnLN?4HJ9U6JRP`t6Ch?m()Xo@)k@y} zfv@-*Oxf<=TU!1`%m__eZhdMr*xx`2Yu%9Dr8|PNO%FO84OgSB>lJvTY`KNp`KQrO*_&y+_gH6$mDeZMF_S0A;? zYE(zJZZX3l_2f}0MKiYPHbMN{H`?{&uyd=Cs%_^{dKK)dN9I8#cmBJizwn|Ss-=6d zo&WXs7?X*@!2B7mU}M4ejKcnoP55Eqoy{P)?FUMUQ@UwhbCVNZn-4i`&s9eJ-O%7E boJ1`da_23!Xo@2?JpPq~x7bP9Qi2LXR0c+( From 1ff8ad64fcd23d55ca873271a1d86a60502d0e55 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 10:02:55 +0200 Subject: [PATCH 083/128] Add new Ansible vault --- ansible/vault/gesis-production.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 ansible/vault/gesis-production.yml diff --git a/ansible/vault/gesis-production.yml b/ansible/vault/gesis-production.yml new file mode 100644 index 000000000..0576d46f5 --- /dev/null +++ b/ansible/vault/gesis-production.yml @@ -0,0 +1,18 @@ +$ANSIBLE_VAULT;1.1;AES256 +32626233376562376639323233666538613863613765326261366535656434663931306235623132 +3561333630333337376461663662663165396630303962310a386331373832366237653436643836 +38666333643435393864666135303731663732343030336561656631663861303338613461343561 +3132653334336139610a336364343431376537316532626332646438656334646331663330646632 +33313237303330346462616562313564623732653435313365333166376162313061656131626536 +31356434663062626633616234393165323632376231656161303563633436396230363533643130 +62653435623037633461623134393132383833306563313938323338633232633363376466393064 +30636134346636616533333935663565336134303063646332633863626230616662643431656539 +30353664633961633263333435336232663538393431316662353336666365373066323066633131 +63613562663466343865306532333565363362386235643962343234613562303164303638623365 +63386635316531356238326364376334663934316661336537663561623664306133356134363661 +32353133333736613063363130303761363966653562613631623436333236366334303030303938 +63613238656662343037373932333933396538376565646434316530616461303032326263646161 +35353337343065343465666538346531633164623932393935316666326337303133613134373835 +33323561663337313230656136376561373665306161353338373333333134313464343266373365 +64333130393738656331666165383963613139613766363732306230393764623866653330373764 +6537 From a189664d37ce2ad29cc48f6b2d872ca53884d846 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 10:08:20 +0200 Subject: [PATCH 084/128] Restore GitLab CI --- .gitlab-ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30c7537a3..85bf3b0cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,15 +70,15 @@ include: --values ./secrets/config/common/cryptnono.yaml \ --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml -# gesis helm stage deploy: -# resource_group: stage -# stage: deploy-stage-helm -# variables: -# HELM_ENVIRONMENT: stage -# extends: -# - .gesis helm deploy +gesis helm stage deploy: + resource_group: stage + stage: deploy-stage-helm + variables: + HELM_ENVIRONMENT: stage + extends: + - .gesis helm deploy -# smoke test to stage cluster: -# stage: test-stage -# script: -# - curl https://notebooks-test.gesis.org/binder/ \ No newline at end of file +smoke test to stage cluster: + stage: test-stage + script: + - curl https://notebooks-test.gesis.org/binder/ \ No newline at end of file From 3267f34ec48253de9a37c3746db172acae269aba Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 10:37:06 +0200 Subject: [PATCH 085/128] Add secret volume to deployment --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 85bf3b0cb..f0185680f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,7 @@ include: --values ./config/gesis-${HELM_ENVIRONMENT}.yaml \ --values ./secrets/config/common/common.yaml \ --values ./secrets/config/common/cryptnono.yaml \ + --values ./secrets/config/common/gesis.yaml \ --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml - | helm upgrade \ @@ -68,6 +69,7 @@ include: --values ./config/gesis-${HELM_ENVIRONMENT}.yaml \ --values ./secrets/config/common/common.yaml \ --values ./secrets/config/common/cryptnono.yaml \ + --values ./secrets/config/common/gesis.yaml \ --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml gesis helm stage deploy: From 7e547dd45817feb2fc4785672bb75897f25412eb Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 13:03:03 +0200 Subject: [PATCH 086/128] Add MetalLB to deployment --- .../roles/k8s-control-panel/tasks/main.yml | 3 +++ .../roles/k8s-control-panel/tasks/metallb.yml | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 ansible/roles/k8s-control-panel/tasks/metallb.yml diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 4b14de201..c0e156b8a 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -164,3 +164,6 @@ name: "remove timeout" job: "python3 /home/ansible/bin/kill-after-timeout-pods.py --verbose >> /home/ansible/kill-after-timeout-pods.log 2>&1" minute: "*/5" +- name: Add MetalLB to Kubernetes cluster + ansible.builtin.import_tasks: + file: metallb.yml diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml new file mode 100644 index 000000000..a952dc642 --- /dev/null +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -0,0 +1,25 @@ +- name: Add a MetalLB Helm repository + kubernetes.core.helm_repository: + repo_name: metallb + repo_url: https://metallb.github.io/metallb +- name: Create MetalLB Kubernetes namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: metallb + labels: + # Required labels + # https://metallb.universe.tf/installation/#installation-with-helm + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged +- name: Deploy MetalLB + kubernetes.core.helm: + release_name: metallb + release_namespace: metallb + chart_ref: metallb/metallb + create_namespace: false + history_max: 3 From b9e64794059de26453b2e225ed02ddcfd0f5e698 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 13:31:34 +0200 Subject: [PATCH 087/128] Add IP Address Pool --- ansible/roles/k8s-control-panel/tasks/metallb.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index a952dc642..e87c2480c 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -16,6 +16,20 @@ pod-security.kubernetes.io/enforce: privileged pod-security.kubernetes.io/audit: privileged pod-security.kubernetes.io/warn: privileged +- name: Create MetalLB Kubernetes IP Address Pool + kubernetes.core.k8s: + state: present + definition: + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: first-pool + namespace: metallb-system + spec: + addresses: + {% for host in groups['all'] %} + - {{ hostvars[host]['ansible_facts']['eth0']['ipv4']['address'] }} + {% endfor %} - name: Deploy MetalLB kubernetes.core.helm: release_name: metallb From 6ed8984fd33c9e7ebfb0d14f93477e11415ab3d5 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 13:36:04 +0200 Subject: [PATCH 088/128] Fix MetalLB IP Address Pool --- ansible/roles/k8s-control-panel/tasks/metallb.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index e87c2480c..455612ce3 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -27,9 +27,7 @@ namespace: metallb-system spec: addresses: - {% for host in groups['all'] %} - - {{ hostvars[host]['ansible_facts']['eth0']['ipv4']['address'] }} - {% endfor %} + - 192.168.100.0/24 - name: Deploy MetalLB kubernetes.core.helm: release_name: metallb From 6ba5968bbae3b766f92cf44f6ab70908010f3e58 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 13:39:35 +0200 Subject: [PATCH 089/128] Add MetalLb advertisement --- .../roles/k8s-control-panel/tasks/metallb.yml | 30 +++++++++++++------ ansible/roles/k8s-control-panel/vars/main.yml | 1 + 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index 455612ce3..a0decc4f9 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -16,6 +16,13 @@ pod-security.kubernetes.io/enforce: privileged pod-security.kubernetes.io/audit: privileged pod-security.kubernetes.io/warn: privileged +- name: Deploy MetalLB + kubernetes.core.helm: + release_name: metallb + release_namespace: metallb + chart_ref: metallb/metallb + create_namespace: false + history_max: 3 - name: Create MetalLB Kubernetes IP Address Pool kubernetes.core.k8s: state: present @@ -23,15 +30,20 @@ apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: - name: first-pool - namespace: metallb-system + name: '{{ k8s_control_panel_metallb_ip_address_pool }}' + namespace: metallb spec: addresses: - 192.168.100.0/24 -- name: Deploy MetalLB - kubernetes.core.helm: - release_name: metallb - release_namespace: metallb - chart_ref: metallb/metallb - create_namespace: false - history_max: 3 +- name: Configure L2 Advertisement for MetalLB + kubernetes.core.k8s: + state: present + definition: + apiVersion: metallb.io/v1beta1 + kind: L2Advertisement + metadata: + name: '{{ k8s_control_panel_metallb_ip_address_pool }}-l2-advertisement' + namespace: metallb + spec: + ipAddressPools: + - '{{ k8s_control_panel_metallb_ip_address_pool }}' diff --git a/ansible/roles/k8s-control-panel/vars/main.yml b/ansible/roles/k8s-control-panel/vars/main.yml index daa8e21d2..f1550ee34 100644 --- a/ansible/roles/k8s-control-panel/vars/main.yml +++ b/ansible/roles/k8s-control-panel/vars/main.yml @@ -1 +1,2 @@ k8s_control_panel_calico_version: "3.28.2" +k8s_control_panel_metallb_ip_address_pool: "gesis" From 25b932ae561e85587df44ba44e608f0e6f431fc1 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 15:39:04 +0200 Subject: [PATCH 090/128] Use variable for IP pool --- ansible/roles/k8s-control-panel/tasks/cni.yml | 2 +- ansible/roles/k8s-control-panel/tasks/metallb.yml | 2 +- .../calico/custom-resources.yaml.jinja} | 2 +- ansible/roles/k8s-control-panel/vars/main.yml | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) rename ansible/roles/k8s-control-panel/{files/calico/custom-resources.yaml => templates/calico/custom-resources.yaml.jinja} (94%) diff --git a/ansible/roles/k8s-control-panel/tasks/cni.yml b/ansible/roles/k8s-control-panel/tasks/cni.yml index 7a091d9e3..fd52c2458 100644 --- a/ansible/roles/k8s-control-panel/tasks/cni.yml +++ b/ansible/roles/k8s-control-panel/tasks/cni.yml @@ -10,7 +10,7 @@ kubernetes.core.k8s: state: present # A local copy of https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/custom-resources.yaml - definition: "{{ lookup('file', '{{ role_path }}/files/calico/custom-resources.yaml') | from_yaml_all }}" + definition: "{{ lookup('ansible.builtin.template', '{{ role_path }}/templates/calico/custom-resources.yaml.jinja') | from_yaml_all }}" - name: Install Cert Manager kubernetes.core.k8s: state: present diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index a0decc4f9..a7e97fa30 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -34,7 +34,7 @@ namespace: metallb spec: addresses: - - 192.168.100.0/24 + - '{{ k8s_control_panel_cidr }}' - name: Configure L2 Advertisement for MetalLB kubernetes.core.k8s: state: present diff --git a/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml b/ansible/roles/k8s-control-panel/templates/calico/custom-resources.yaml.jinja similarity index 94% rename from ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml rename to ansible/roles/k8s-control-panel/templates/calico/custom-resources.yaml.jinja index 746e1d94d..df37ae858 100644 --- a/ansible/roles/k8s-control-panel/files/calico/custom-resources.yaml +++ b/ansible/roles/k8s-control-panel/templates/calico/custom-resources.yaml.jinja @@ -10,7 +10,7 @@ spec: ipPools: - name: default-ipv4-ippool blockSize: 26 - cidr: 10.244.0.0/16 + cidr: '{{ k8s_control_panel_cidr }}' encapsulation: VXLANCrossSubnet natOutgoing: Enabled nodeSelector: all() diff --git a/ansible/roles/k8s-control-panel/vars/main.yml b/ansible/roles/k8s-control-panel/vars/main.yml index f1550ee34..3474a6307 100644 --- a/ansible/roles/k8s-control-panel/vars/main.yml +++ b/ansible/roles/k8s-control-panel/vars/main.yml @@ -1,2 +1,3 @@ k8s_control_panel_calico_version: "3.28.2" -k8s_control_panel_metallb_ip_address_pool: "gesis" +k8s_control_panel_cidr: "10.244.0.0/16" +k8s_control_panel_metallb_ip_address_pool_name: "gesis" From 49e0f6ff957f2f1dee74357edcfcaad2aadddb0b Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 15:39:19 +0200 Subject: [PATCH 091/128] Disable MetalLB speaker --- ansible/roles/k8s-control-panel/tasks/metallb.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index a7e97fa30..53ce9c4fd 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -21,6 +21,9 @@ release_name: metallb release_namespace: metallb chart_ref: metallb/metallb + release_values: + speaker: + enabled: false create_namespace: false history_max: 3 - name: Create MetalLB Kubernetes IP Address Pool From 47572e25d699325b305f0de5fd86d4ec55e38939 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 15:42:32 +0200 Subject: [PATCH 092/128] Fix MetalLB name --- ansible/roles/k8s-control-panel/tasks/metallb.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index 53ce9c4fd..88d55dd63 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -33,7 +33,7 @@ apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: - name: '{{ k8s_control_panel_metallb_ip_address_pool }}' + name: '{{ k8s_control_panel_metallb_ip_address_pool_name }}' namespace: metallb spec: addresses: @@ -45,8 +45,8 @@ apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: - name: '{{ k8s_control_panel_metallb_ip_address_pool }}-l2-advertisement' + name: '{{ k8s_control_panel_metallb_ip_address_pool_name }}-l2-advertisement' namespace: metallb spec: ipAddressPools: - - '{{ k8s_control_panel_metallb_ip_address_pool }}' + - '{{ k8s_control_panel_metallb_ip_address_pool_name }}' From bb5db91cdaff8fc8a8c271620fa8c4efbe33409d Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 15:43:49 +0200 Subject: [PATCH 093/128] Enable MetalDB speakers --- ansible/roles/k8s-control-panel/tasks/metallb.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index 88d55dd63..693850c87 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -21,9 +21,6 @@ release_name: metallb release_namespace: metallb chart_ref: metallb/metallb - release_values: - speaker: - enabled: false create_namespace: false history_max: 3 - name: Create MetalLB Kubernetes IP Address Pool From 93ec46d414af24051f2b30882f2e365b53dfae5c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 16:07:17 +0200 Subject: [PATCH 094/128] Use IP range for MetalLB --- ansible/inventories/gesis-stage | 3 +++ ansible/roles/k8s-control-panel/tasks/metallb.yml | 3 ++- config/gesis-stage.yaml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ansible/inventories/gesis-stage b/ansible/inventories/gesis-stage index 66beeb4d3..d97fa2add 100644 --- a/ansible/inventories/gesis-stage +++ b/ansible/inventories/gesis-stage @@ -9,6 +9,9 @@ svko-k8s-test03 ansible_host=194.95.75.23 ansible_ssh_user=ansible ansible_becom INVENTORY_NAME=stage K8S_CONTROL_PLANE_ENDPOINT=194.95.75.21 K8S_CONTROL_PLANE_ALIAS=svko-k8s-test01 +; Replace this variable with a filter +; This must match the group ingress +K8S_INGRESS=194.95.75.22 [notebooks_gesis_org] ; svko-css-backup-node diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index 693850c87..894138f8e 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -34,7 +34,8 @@ namespace: metallb spec: addresses: - - '{{ k8s_control_panel_cidr }}' + # TODO Use Jinja filter to automate this. + - '{{ K8S_INGRESS }}-{{ K8S_INGRESS }}' - name: Configure L2 Advertisement for MetalLB kubernetes.core.k8s: state: present diff --git a/config/gesis-stage.yaml b/config/gesis-stage.yaml index af7970be1..638c366d8 100644 --- a/config/gesis-stage.yaml +++ b/config/gesis-stage.yaml @@ -142,7 +142,7 @@ ingress-nginx: enabled: true service: externalTrafficPolicy: null - type: ClusterIP + type: LoadBalancer prometheus: enabled: true server: From 28c0d8e4d69c67719393d8eb22531b70f77a4898 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 24 Sep 2024 18:09:25 +0200 Subject: [PATCH 095/128] Add placeholder for GESIS documentation --- docs/source/deployment/gesis-diagram.svg | 3 +++ docs/source/deployment/gesis.md | 28 ++++++++++++++++++++++++ docs/source/deployment/index.rst | 1 + 3 files changed, 32 insertions(+) create mode 100755 docs/source/deployment/gesis-diagram.svg create mode 100644 docs/source/deployment/gesis.md diff --git a/docs/source/deployment/gesis-diagram.svg b/docs/source/deployment/gesis-diagram.svg new file mode 100755 index 000000000..f27e6965f --- /dev/null +++ b/docs/source/deployment/gesis-diagram.svg @@ -0,0 +1,3 @@ + + +notebooks.gesis.orgGoogle CloudGESIS GitLabGitHub ActionsGitHubnotebooks.gesis.orgGoogle CloudGESIS GitLabGitHub ActionsGitHubDevelopergit commitgit pushtriggervalidationhelm upgradetriggervalidationhelm upgradeDeveloper \ No newline at end of file diff --git a/docs/source/deployment/gesis.md b/docs/source/deployment/gesis.md new file mode 100644 index 000000000..ad7e77f82 --- /dev/null +++ b/docs/source/deployment/gesis.md @@ -0,0 +1,28 @@ +# How to deploy a change to notebooks.gesis.org? + +[GESIS Leibniz Institute for the Social Sciences](https://www.gesis.org) is a member of the [mybinder.org federation](https://mybinder.readthedocs.io/en/latest/about/status.html). GESIS has on-premise servers and use it for the mybinder.org server. The use of on-premise servers requires a separate deployment because the access to the servers using SSH requires the tunelling using a VPN. + + + +![Sequence diagram illustrating the deployment.](./gesis-diagram.svg) + +## Virtual Private Server configuration with Ansible + +We use [Ansible](https://www.ansible.com/) to automate the configuration of the virtual private server (VPS) provided by GESIS. After a successful configuration, we will have a operational Kubernetes cluster to deploy mybinder.org. \ No newline at end of file diff --git a/docs/source/deployment/index.rst b/docs/source/deployment/index.rst index 5ab6e4f25..b9a7224f0 100644 --- a/docs/source/deployment/index.rst +++ b/docs/source/deployment/index.rst @@ -8,3 +8,4 @@ Deployment and Operation prereqs how what + gesis From d7c348d6705ff62d407c4cc1e4748f76cb1e6fb3 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Wed, 25 Sep 2024 10:08:15 +0200 Subject: [PATCH 096/128] Document load balancer --- docs/source/deployment/gesis-load-balancer.drawio.svg | 4 ++++ docs/source/deployment/gesis.md | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100755 docs/source/deployment/gesis-load-balancer.drawio.svg diff --git a/docs/source/deployment/gesis-load-balancer.drawio.svg b/docs/source/deployment/gesis-load-balancer.drawio.svg new file mode 100755 index 000000000..7d60733f8 --- /dev/null +++ b/docs/source/deployment/gesis-load-balancer.drawio.svg @@ -0,0 +1,4 @@ + + + +
User
GESIS on-premise physical server
Virtual Private Server 1
Kubernetes Node
Virtual Private Server 2
Kubernetes Node
Ingress NGINX
Controller
mybinder.org
 \ No newline at end of file diff --git a/docs/source/deployment/gesis.md b/docs/source/deployment/gesis.md index ad7e77f82..91b047c44 100644 --- a/docs/source/deployment/gesis.md +++ b/docs/source/deployment/gesis.md @@ -23,6 +23,12 @@ sequenceDiagram ![Sequence diagram illustrating the deployment.](./gesis-diagram.svg) +## Kubernetes on bare metal + +Cloud environments provide a load balancer to the Kubernetes clusters. Unfortunately, Kubernetes cluster does not includes a default implementation of a load balancer for the scenario that it is running on bare metal. Because of this, the deployment of mybinder.org to GESIS servers must include the configuration of a load balancer. We are using [MetalLB](https://metallb.universe.tf/) with [Ingress NGINX Controller](https://kubernetes.github.io/ingress-nginx/). + +![Sequence diagram illustrating the load balancer.](./gesis-load-balancer.drawio.svg) + ## Virtual Private Server configuration with Ansible We use [Ansible](https://www.ansible.com/) to automate the configuration of the virtual private server (VPS) provided by GESIS. After a successful configuration, we will have a operational Kubernetes cluster to deploy mybinder.org. \ No newline at end of file From 5b2e10e22ddfdd880c1fb5bcd564bcacc8901ea5 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Wed, 25 Sep 2024 10:12:48 +0200 Subject: [PATCH 097/128] Mention GitLab CI --- docs/source/deployment/gesis.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/deployment/gesis.md b/docs/source/deployment/gesis.md index 91b047c44..548a42adb 100644 --- a/docs/source/deployment/gesis.md +++ b/docs/source/deployment/gesis.md @@ -23,6 +23,12 @@ sequenceDiagram ![Sequence diagram illustrating the deployment.](./gesis-diagram.svg) +## GESIS GitLab CI/CD Server + +GESIS GitLab server runs [GitLab Community Edition v16.11.6](https://gitlab.com/gitlab-org/gitlab-foss/-/tags/v16.11.6) with [continuous integration (CI) and continuous delivery (CD)](https://about.gitlab.com/topics/ci-cd/) enable. + +The CI/CD jobs are defined in [`.gitlab-ci.yml`](https://github.com/jupyterhub/mybinder.org-deploy/tree/main/.gitlab-ci.yml). + ## Kubernetes on bare metal Cloud environments provide a load balancer to the Kubernetes clusters. Unfortunately, Kubernetes cluster does not includes a default implementation of a load balancer for the scenario that it is running on bare metal. Because of this, the deployment of mybinder.org to GESIS servers must include the configuration of a load balancer. We are using [MetalLB](https://metallb.universe.tf/) with [Ingress NGINX Controller](https://kubernetes.github.io/ingress-nginx/). From 674798fcbc36fbb189621473afe70dfe627e7151 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Wed, 25 Sep 2024 10:27:52 +0200 Subject: [PATCH 098/128] Update GitLab CI --- .gitlab-ci.yml | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0185680f..e7d2e0a17 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,29 @@ variables: GIT_STRATEGY: clone GIT_CLEAN_FLAGS: "-ffdx" +.gesis-manual-web: + rules: + - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == 'web' + when: manual + allow_failure: true + +.geis-merge-request: + rules: + - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - .gitlab.yml + when: manual + - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - ansible/**/* + - mybinder/**/* + - config/**/* + - secrets/**/* + +.geis-push-main: + rules: + - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'main' + stages: - build - deploy-stage-ansible @@ -75,12 +98,27 @@ include: gesis helm stage deploy: resource_group: stage stage: deploy-stage-helm + rules: + - !reference [.manual-web, rules] + - !reference [.geis-merge-request, rules] + - !reference [.geis-push-main, rules] + variables: + HELM_ENVIRONMENT: stage + extends: + - .gesis helm deploy + +gesis helm production deploy: + resource_group: production + stage: deploy-production-helm + rules: + - !reference [.manual-web, rules] + - !reference [.geis-push-main, rules] variables: HELM_ENVIRONMENT: stage extends: - .gesis helm deploy -smoke test to stage cluster: +smoke test after stage deploy: stage: test-stage script: - curl https://notebooks-test.gesis.org/binder/ \ No newline at end of file From 26dc100da60642ff7234e31dad0376670f4eb19a Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Wed, 25 Sep 2024 10:45:04 +0200 Subject: [PATCH 099/128] Fix GitLab CI --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e7d2e0a17..f482a385e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -99,7 +99,7 @@ gesis helm stage deploy: resource_group: stage stage: deploy-stage-helm rules: - - !reference [.manual-web, rules] + - !reference [.gesis-manual-web, rules] - !reference [.geis-merge-request, rules] - !reference [.geis-push-main, rules] variables: @@ -111,7 +111,7 @@ gesis helm production deploy: resource_group: production stage: deploy-production-helm rules: - - !reference [.manual-web, rules] + - !reference [.gesis-manual-web, rules] - !reference [.geis-push-main, rules] variables: HELM_ENVIRONMENT: stage @@ -120,5 +120,9 @@ gesis helm production deploy: smoke test after stage deploy: stage: test-stage + rules: + - !reference [.gesis-manual-web, rules] + - !reference [.geis-merge-request, rules] + - !reference [.geis-push-main, rules] script: - curl https://notebooks-test.gesis.org/binder/ \ No newline at end of file From 5ce49a9ae5652886d285d823ae5f338c4e58fb67 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Wed, 25 Sep 2024 10:47:24 +0200 Subject: [PATCH 100/128] Fix stage names in GitLab CI --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f482a385e..052c71392 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,8 +30,8 @@ stages: - deploy-stage-ansible - deploy-stage-helm - test-stage - - deploy-prod-nginx - - deploy-prod-helm + - deploy-production-ansible + - deploy-production-helm include: - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.6 From e240f58f9dc2d65ddc61fb7bddd592b99e1f80bc Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Wed, 25 Sep 2024 13:59:47 +0200 Subject: [PATCH 101/128] pixi run pre-commit run -a --- .editorconfig | 2 +- .gitlab-ci.yml | 6 +++--- .gitlab/agents/stage/config.yaml | 2 +- ansible/roles/k8s-common/tasks/k8s-repository.yml | 4 ++-- ansible/roles/k8s-common/tasks/main.yml | 12 ++++++------ .../files/cron/kill-after-timeout-pods.py | 6 ++---- .../files/cron/kill-succeeded-pods.py | 4 +--- .../files/usr/bin/orc2-fix-dind-bot.py | 7 +++---- ansible/roles/k8s-control-panel/tasks/main.yml | 6 +++--- ansible/roles/k8s-control-panel/tasks/metallb.yml | 8 ++++---- ansible/roles/k8s-worker/tasks/main.yml | 2 +- docs/source/deployment/gesis.md | 2 +- 12 files changed, 28 insertions(+), 33 deletions(-) diff --git a/.editorconfig b/.editorconfig index 216de0c51..5afdf958f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,4 +5,4 @@ charset = utf-8 end_of_line = lf indent_size = 2 indent_style = space -insert_final_newline = true \ No newline at end of file +insert_final_newline = true diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 052c71392..b2952d7e1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,7 +13,7 @@ variables: - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "merge_request_event" changes: - .gitlab.yml - when: manual + when: manual - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "merge_request_event" changes: - ansible/**/* @@ -49,7 +49,7 @@ include: ssh-key-type: ed25519 .gesis helm deploy: - image: + image: name: docker-private.gesis.intra/gesis/ilcm/orc2/k8s:latest entrypoint: [""] variables: @@ -125,4 +125,4 @@ smoke test after stage deploy: - !reference [.geis-merge-request, rules] - !reference [.geis-push-main, rules] script: - - curl https://notebooks-test.gesis.org/binder/ \ No newline at end of file + - curl https://notebooks-test.gesis.org/binder/ diff --git a/.gitlab/agents/stage/config.yaml b/.gitlab/agents/stage/config.yaml index 59f3bb059..c402b2fa8 100644 --- a/.gitlab/agents/stage/config.yaml +++ b/.gitlab/agents/stage/config.yaml @@ -1,3 +1,3 @@ ci_access: projects: - - id: methods-hub/interactive-environment \ No newline at end of file + - id: methods-hub/interactive-environment diff --git a/ansible/roles/k8s-common/tasks/k8s-repository.yml b/ansible/roles/k8s-common/tasks/k8s-repository.yml index 2fc66d81b..fc20cd955 100644 --- a/ansible/roles/k8s-common/tasks/k8s-repository.yml +++ b/ansible/roles/k8s-common/tasks/k8s-repository.yml @@ -26,7 +26,7 @@ ansible.builtin.get_url: url: "https://pkgs.k8s.io/core:/stable:/v{{ k8s_common_kubernetes_version }}/deb/Release.key" dest: /tmp/kubernetes-archive-keyring.asc - mode: '0644' + mode: "0644" force: true - name: Convert the public GPG key to binary ansible.builtin.command: @@ -43,7 +43,7 @@ src: /tmp/kubernetes.gpg dest: /etc/apt/keyrings/kubernetes.gpg remote_src: true - mode: '0644' + mode: "0644" - name: Add Kubernetes repository ansible.builtin.apt_repository: repo: "deb [signed-by=/etc/apt/keyrings/kubernetes.gpg] https://pkgs.k8s.io/core:/stable:/v{{ k8s_common_kubernetes_version }}/deb/ /" diff --git a/ansible/roles/k8s-common/tasks/main.yml b/ansible/roles/k8s-common/tasks/main.yml index e8aac97b6..20045b981 100644 --- a/ansible/roles/k8s-common/tasks/main.yml +++ b/ansible/roles/k8s-common/tasks/main.yml @@ -11,7 +11,7 @@ - name: Ensure DOCKER_CLIENT_TIMEOUT is set ansible.builtin.lineinfile: path: /etc/environment - regexp: '^DOCKER_CLIENT_TIMEOUT=' + regexp: "^DOCKER_CLIENT_TIMEOUT=" line: DOCKER_CLIENT_TIMEOUT=180 - name: Disable SWAP since kubernetes can't work with swap enabled ansible.builtin.command: swapoff -a @@ -27,17 +27,17 @@ - name: Allow IP forward ansible.posix.sysctl: name: net.ipv4.ip_forward - value: '1' + value: "1" state: present - name: Set inotify max user instances ansible.posix.sysctl: name: fs.inotify.max_user_instances - value: '1280' + value: "1280" state: present - name: Set inotify max user watches ansible.posix.sysctl: name: fs.inotify.max_user_watches - value: '655360' + value: "655360" state: present - name: Create directory for Persistent Volume ansible.builtin.import_tasks: @@ -46,7 +46,7 @@ ansible.builtin.get_url: url: https://download.docker.com/linux/ubuntu/gpg dest: /etc/apt/trusted.gpg.d/docker.asc - mode: '0644' + mode: "0644" force: true - name: Add Docker repository ansible.builtin.apt_repository: @@ -57,7 +57,7 @@ ansible.builtin.get_url: url: https://baltocdn.com/helm/signing.asc dest: /etc/apt/trusted.gpg.d/helm.asc - mode: '0644' + mode: "0644" force: true - name: Add Helm repository ansible.builtin.apt_repository: diff --git a/ansible/roles/k8s-control-panel/files/cron/kill-after-timeout-pods.py b/ansible/roles/k8s-control-panel/files/cron/kill-after-timeout-pods.py index e52f7d828..67fbca9ab 100644 --- a/ansible/roles/k8s-control-panel/files/cron/kill-after-timeout-pods.py +++ b/ansible/roles/k8s-control-panel/files/cron/kill-after-timeout-pods.py @@ -1,8 +1,8 @@ """Kill pods in Kubernetes cluster after timeout""" import argparse -import logging import datetime +import logging from kubernetes import client, config @@ -48,9 +48,7 @@ def kill_pod(pod): api_response = v1.delete_namespaced_pod(pod.metadata.name, NAMESPACE) logger.info("Pod %s deleted.", api_response.metadata.name) except client.exceptions.ApiException as exception: - logger.info( - "Fail to delete pod %s due %s", pod.metadata.name, exception - ) + logger.info("Fail to delete pod %s due %s", pod.metadata.name, exception) def kill_timed_out_pods(): diff --git a/ansible/roles/k8s-control-panel/files/cron/kill-succeeded-pods.py b/ansible/roles/k8s-control-panel/files/cron/kill-succeeded-pods.py index deb909c1d..3b75368e2 100644 --- a/ansible/roles/k8s-control-panel/files/cron/kill-succeeded-pods.py +++ b/ansible/roles/k8s-control-panel/files/cron/kill-succeeded-pods.py @@ -34,9 +34,7 @@ def kill_pod(pod): api_response = v1.delete_namespaced_pod(pod.metadata.name, NAMESPACE) logger.info("Pod %s deleted.", api_response.metadata.name) except client.exceptions.ApiException as exception: - logger.info( - "Fail to delete pod %s due %s", pod.metadata.name, exception - ) + logger.info("Fail to delete pod %s due %s", pod.metadata.name, exception) def kill_succeeded_pods(): diff --git a/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-dind-bot.py b/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-dind-bot.py index daf580ae7..f6e749b30 100644 --- a/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-dind-bot.py +++ b/ansible/roles/k8s-control-panel/files/usr/bin/orc2-fix-dind-bot.py @@ -5,10 +5,9 @@ import logging import os -from kubernetes import client, config, watch - -from invoke import Responder from fabric import Connection +from invoke import Responder +from kubernetes import client, config, watch logging.basicConfig( format="%(asctime)s %(levelname)-8s | %(message)s", datefmt="%Y-%m-%d %H:%M:%S" @@ -99,7 +98,7 @@ def monitor_cluster(): logger.info( "Fail to delete pod %s due %s", pod_name, exception ) - + elif event["object"].type == "Normal": logger.debug( "Found Normal event in %s ... skipping!", diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index c0e156b8a..2143c6d49 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -73,7 +73,7 @@ - name: Copy join command to local file ansible.builtin.fetch: src: /tmp/kubernetes_join_command - dest: '{{ ANSIBLE_CONTROL_NODE_TMP }}' + dest: "{{ ANSIBLE_CONTROL_NODE_TMP }}" - name: Add Container Network Interface (CNI) to Kubernetes cluster ansible.builtin.import_tasks: file: cni.yml @@ -83,13 +83,13 @@ repo_url: https://charts.gitlab.io - name: Deploy GitLab agent kubernetes.core.helm: - name: 'gitlab-agent-{{ inventory_file | basename }}' + name: "gitlab-agent-{{ inventory_file | basename }}" chart_ref: gitlab/gitlab-agent release_namespace: gitlab-agent dependency_update: true create_namespace: true set_values: - - value: 'config.token={{ GITLAB_K8S_TOKEN }}' + - value: "config.token={{ GITLAB_K8S_TOKEN }}" - value: config.kasAddress=wss://git.gesis.org/-/kubernetes-agent/ - name: Copy orc2-fix-jupyterhub-bot Python script ansible.builtin.copy: diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index 894138f8e..9088a9f27 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -30,12 +30,12 @@ apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: - name: '{{ k8s_control_panel_metallb_ip_address_pool_name }}' + name: "{{ k8s_control_panel_metallb_ip_address_pool_name }}" namespace: metallb spec: addresses: # TODO Use Jinja filter to automate this. - - '{{ K8S_INGRESS }}-{{ K8S_INGRESS }}' + - "{{ K8S_INGRESS }}-{{ K8S_INGRESS }}" - name: Configure L2 Advertisement for MetalLB kubernetes.core.k8s: state: present @@ -43,8 +43,8 @@ apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: - name: '{{ k8s_control_panel_metallb_ip_address_pool_name }}-l2-advertisement' + name: "{{ k8s_control_panel_metallb_ip_address_pool_name }}-l2-advertisement" namespace: metallb spec: ipAddressPools: - - '{{ k8s_control_panel_metallb_ip_address_pool_name }}' + - "{{ k8s_control_panel_metallb_ip_address_pool_name }}" diff --git a/ansible/roles/k8s-worker/tasks/main.yml b/ansible/roles/k8s-worker/tasks/main.yml index 7f1d6b455..4fb307a5e 100644 --- a/ansible/roles/k8s-worker/tasks/main.yml +++ b/ansible/roles/k8s-worker/tasks/main.yml @@ -1,6 +1,6 @@ - name: Copy join command ansible.builtin.copy: - src: '{{ ANSIBLE_CONTROL_NODE_TMP }}/{{ K8S_CONTROL_PLANE_ALIAS }}/tmp/kubernetes_join_command' + src: "{{ ANSIBLE_CONTROL_NODE_TMP }}/{{ K8S_CONTROL_PLANE_ALIAS }}/tmp/kubernetes_join_command" dest: /tmp/kubernetes_join_command mode: u=rwx,g=rx,o=rx - name: Attempt to join cluster diff --git a/docs/source/deployment/gesis.md b/docs/source/deployment/gesis.md index 548a42adb..039ccb9ea 100644 --- a/docs/source/deployment/gesis.md +++ b/docs/source/deployment/gesis.md @@ -37,4 +37,4 @@ Cloud environments provide a load balancer to the Kubernetes clusters. Unfortuna ## Virtual Private Server configuration with Ansible -We use [Ansible](https://www.ansible.com/) to automate the configuration of the virtual private server (VPS) provided by GESIS. After a successful configuration, we will have a operational Kubernetes cluster to deploy mybinder.org. \ No newline at end of file +We use [Ansible](https://www.ansible.com/) to automate the configuration of the virtual private server (VPS) provided by GESIS. After a successful configuration, we will have a operational Kubernetes cluster to deploy mybinder.org. From bbd180ad2d2456b66aac30641c18d452fa686984 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 17:01:47 +0100 Subject: [PATCH 102/128] Rename stages --- .gitlab-ci.yml | 38 ++++++++++--------- .../{gesis-stage => gesis-acceptance } | 19 ++++------ .../{gesis-stage.yml => gesis-acceptance.yml} | 0 3 files changed, 28 insertions(+), 29 deletions(-) rename ansible/inventories/{gesis-stage => gesis-acceptance } (70%) rename ansible/vault/{gesis-stage.yml => gesis-acceptance.yml} (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b2952d7e1..105407275 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,14 @@ variables: GIT_STRATEGY: clone GIT_CLEAN_FLAGS: "-ffdx" +acceptances: + - lint + - deploy-acceptance-ansible + - deploy-acceptance-helm + - test-acceptance + - deploy-production-nginx + - deploy-production-helm + .gesis-manual-web: rules: - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == 'web' @@ -25,33 +33,29 @@ variables: rules: - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'main' -stages: - - build - - deploy-stage-ansible - - deploy-stage-helm - - test-stage - - deploy-production-ansible - - deploy-production-helm - include: - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.6 inputs: - stage: build + stage: lint dir: ansible - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-deploy@10.2.6 inputs: - stage: deploy-stage-ansible + stage: deploy-acceptance-ansible dir: ansible - inventory: gesis-stage + inventory: gesis-acceptance playbook: gesis.yml ssh-user: ansible ssh-key-type: ed25519 + rules: + - if: CI_SERVER_HOST == "git.gesis.org" .gesis helm deploy: image: name: docker-private.gesis.intra/gesis/ilcm/orc2/k8s:latest entrypoint: [""] + rules: + - if: CI_SERVER_HOST == "git.gesis.org" variables: HELM_ENVIRONMENT: template script: @@ -95,15 +99,15 @@ include: --values ./secrets/config/common/gesis.yaml \ --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml -gesis helm stage deploy: - resource_group: stage - stage: deploy-stage-helm +gesis helm acceptance deploy: + resource_group: acceptance + stage: deploy-acceptance-helm rules: - !reference [.gesis-manual-web, rules] - !reference [.geis-merge-request, rules] - !reference [.geis-push-main, rules] variables: - HELM_ENVIRONMENT: stage + HELM_ENVIRONMENT: acceptance extends: - .gesis helm deploy @@ -118,8 +122,8 @@ gesis helm production deploy: extends: - .gesis helm deploy -smoke test after stage deploy: - stage: test-stage +smoke test to acceptance cluster: + stage: test-acceptance rules: - !reference [.gesis-manual-web, rules] - !reference [.geis-merge-request, rules] diff --git a/ansible/inventories/gesis-stage b/ansible/inventories/gesis-acceptance similarity index 70% rename from ansible/inventories/gesis-stage rename to ansible/inventories/gesis-acceptance index d97fa2add..d284d2397 100644 --- a/ansible/inventories/gesis-stage +++ b/ansible/inventories/gesis-acceptance @@ -1,6 +1,5 @@ [all] -#svko-ilcm04 ansible_host=194.95.75.14 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_14 }}' -; svko-css-backup-node ansible_host=194.95.75.20 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_20 }}' +svko-css-backup-node ansible_host=194.95.75.20 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_20 }}' svko-k8s-test01 ansible_host=194.95.75.21 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_21 }}' svko-k8s-test02 ansible_host=194.95.75.22 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_22 }}' svko-k8s-test03 ansible_host=194.95.75.23 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_23 }}' @@ -14,8 +13,7 @@ K8S_CONTROL_PLANE_ALIAS=svko-k8s-test01 K8S_INGRESS=194.95.75.22 [notebooks_gesis_org] -; svko-css-backup-node -svko-k8s-test02 +svko-css-backup-node [kubernetes_control_panel] svko-k8s-test01 @@ -26,17 +24,14 @@ JUPYTERHUB_CAPACITY_STORAGE=2Gi PROMETHEUS_CAPACITY_STORAGE=15Gi [kubernetes_workers] -#svko-ilcm04 -; svko-css-backup-node svko-k8s-test02 svko-k8s-test03 [ingress] -; svko-css-backup-node -svko-k8s-test02 +svko-css-backup-node [harbor] -; svko-css-backup-node +svko-k8s-test03 [binderhub] svko-k8s-test02 @@ -45,10 +40,10 @@ svko-k8s-test02 svko-k8s-test02 [jupyterhub_single_user] -svko-k8s-test03 +svko-k8s-test02 [prometheus] -; svko-css-backup-node +svko-css-backup-node [grafana] -; svko-css-backup-node +svko-css-backup-node diff --git a/ansible/vault/gesis-stage.yml b/ansible/vault/gesis-acceptance.yml similarity index 100% rename from ansible/vault/gesis-stage.yml rename to ansible/vault/gesis-acceptance.yml From 62c8f19da2ba7aaddad6ce2da9caca4b21d70195 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 17:03:14 +0100 Subject: [PATCH 103/128] Add role for Harbor --- ansible/gesis.yml | 5 + ansible/roles/harbor/tasks/main.yml | 156 ++++++++++++++++++++++++++++ ansible/roles/harbor/vars/main.yml | 6 ++ 3 files changed, 167 insertions(+) create mode 100644 ansible/roles/harbor/tasks/main.yml create mode 100644 ansible/roles/harbor/vars/main.yml diff --git a/ansible/gesis.yml b/ansible/gesis.yml index 430e341b0..afdee1ac4 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -22,6 +22,11 @@ become: true roles: - k8s-pv +- name: Configure Harbor + hosts: kubernetes_control_panel + gather_facts: false + roles: + - harbor - name: Configure JupyterHub workers hosts: jupyterhub_single_user gather_facts: false diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml new file mode 100644 index 000000000..0902075e7 --- /dev/null +++ b/ansible/roles/harbor/tasks/main.yml @@ -0,0 +1,156 @@ +- name: Add harbor's repository + kubernetes.core.helm_repository: + name: harbor + repo_url: https://helm.goharbor.io + +- name: Create a storage for jobservice + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolume + metadata: + name: "{{ harbor_namespace }}-jobservice" + labels: + harbor: jobservice + spec: + capacity: + storage: "{{harbor_jobservice_storage}}" + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Delete + storageClassName: standard + local: + path: /harbor/jobservice + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' + +- name: Create a storage for registry + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolume + metadata: + name: "{{ harbor_namespace }}-registry" + labels: + harbor: registry + spec: + capacity: + storage: "{{harbor_registry_storage}}" + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Delete + storageClassName: standard + local: + path: /harbor/registry + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' + +- name: Create a storage for redis + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolume + metadata: + name: "{{ harbor_namespace }}-registry" + labels: + harbor: registry + spec: + capacity: + storage: "{{harbor_redis_storage}}" + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Delete + storageClassName: standard + local: + path: /harbor/registry + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' + +- name: Create a storage for trivy + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolume + metadata: + name: "{{ harbor_namespace }}-registry" + labels: + harbor: registry + spec: + capacity: + storage: "{{harbor_trivy_storage}}" + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Delete + storageClassName: standard + local: + path: /harbor/registry + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' + +- name: Create a storage for database + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolume + metadata: + name: "{{ harbor_namespace }}-registry" + labels: + harbor: registry + spec: + capacity: + storage: "{{harbor_database_storage}}" + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Delete + storageClassName: standard + local: + path: /harbor/registry + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' + +- name: Deploy harbor + kubernetes.core.helm: + name: harbor + chart_version: 1.16.0 + release_namespace: "{{ harbor_namespace }}" + create_namespace: true diff --git a/ansible/roles/harbor/vars/main.yml b/ansible/roles/harbor/vars/main.yml new file mode 100644 index 000000000..a238de1ed --- /dev/null +++ b/ansible/roles/harbor/vars/main.yml @@ -0,0 +1,6 @@ +harbor_namespace: harbor +harbor_jobservice_storage: 10Gi +harbor_registry_storage: 10Gi +harbor_redis_storage: 10Gi +harbor_trivy_storage: 10Gi +harbor_database_storage: 10Gi From 77c8c5f9bb1443650d54a5da0a49b8e9707abf23 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 17:08:41 +0100 Subject: [PATCH 104/128] Add missing $ to the name of variable --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 105407275..a322ee690 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,7 +48,7 @@ include: ssh-user: ansible ssh-key-type: ed25519 rules: - - if: CI_SERVER_HOST == "git.gesis.org" + - if: $CI_SERVER_HOST == "git.gesis.org" .gesis helm deploy: image: From 29b52e670f9043a62e905db85ff608d7b4d15d57 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 17:12:21 +0100 Subject: [PATCH 105/128] Fix name of GitLab CI definition --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a322ee690..8915568fe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ variables: GIT_STRATEGY: clone GIT_CLEAN_FLAGS: "-ffdx" -acceptances: +stages: - lint - deploy-acceptance-ansible - deploy-acceptance-helm From 6492acf21cfb2d01228f27ac812b29c903094daf Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 17:12:49 +0100 Subject: [PATCH 106/128] Add missing $ to the name of variable --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8915568fe..f0d3744cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,7 @@ include: name: docker-private.gesis.intra/gesis/ilcm/orc2/k8s:latest entrypoint: [""] rules: - - if: CI_SERVER_HOST == "git.gesis.org" + - if: $CI_SERVER_HOST == "git.gesis.org" variables: HELM_ENVIRONMENT: template script: From 834df81e204ba5ff577e499c7ac3450c94f9674c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 17:16:46 +0100 Subject: [PATCH 107/128] Fix lint errors --- ansible/roles/harbor/tasks/main.yml | 72 ++++++++++++++--------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index 0902075e7..73bdc68a9 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -15,10 +15,10 @@ harbor: jobservice spec: capacity: - storage: "{{harbor_jobservice_storage}}" + storage: "{{ harbor_jobservice_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteMany persistentVolumeReclaimPolicy: Delete storageClassName: standard local: @@ -26,11 +26,11 @@ nodeAffinity: required: nodeSelectorTerms: - - matchExpressions: - - key: harbor - operator: In - values: - - 'true' + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' - name: Create a storage for registry kubernetes.core.k8s: @@ -44,10 +44,10 @@ harbor: registry spec: capacity: - storage: "{{harbor_registry_storage}}" + storage: "{{ harbor_registry_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteMany persistentVolumeReclaimPolicy: Delete storageClassName: standard local: @@ -55,11 +55,11 @@ nodeAffinity: required: nodeSelectorTerms: - - matchExpressions: - - key: harbor - operator: In - values: - - 'true' + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' - name: Create a storage for redis kubernetes.core.k8s: @@ -73,10 +73,10 @@ harbor: registry spec: capacity: - storage: "{{harbor_redis_storage}}" + storage: "{{ harbor_redis_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteMany persistentVolumeReclaimPolicy: Delete storageClassName: standard local: @@ -84,11 +84,11 @@ nodeAffinity: required: nodeSelectorTerms: - - matchExpressions: - - key: harbor - operator: In - values: - - 'true' + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' - name: Create a storage for trivy kubernetes.core.k8s: @@ -102,10 +102,10 @@ harbor: registry spec: capacity: - storage: "{{harbor_trivy_storage}}" + storage: "{{ harbor_trivy_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteMany persistentVolumeReclaimPolicy: Delete storageClassName: standard local: @@ -113,11 +113,11 @@ nodeAffinity: required: nodeSelectorTerms: - - matchExpressions: - - key: harbor - operator: In - values: - - 'true' + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' - name: Create a storage for database kubernetes.core.k8s: @@ -131,10 +131,10 @@ harbor: registry spec: capacity: - storage: "{{harbor_database_storage}}" + storage: "{{ harbor_database_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteMany persistentVolumeReclaimPolicy: Delete storageClassName: standard local: @@ -142,15 +142,15 @@ nodeAffinity: required: nodeSelectorTerms: - - matchExpressions: - - key: harbor - operator: In - values: - - 'true' + - matchExpressions: + - key: harbor + operator: In + values: + - 'true' - name: Deploy harbor kubernetes.core.helm: name: harbor - chart_version: 1.16.0 + chart_version: 1.16.0 release_namespace: "{{ harbor_namespace }}" create_namespace: true From cce9d243332b28aedac9ad4318162b2165fb7f5c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 17:22:38 +0100 Subject: [PATCH 108/128] Remove white space from file name --- ansible/inventories/{gesis-acceptance => gesis-acceptance} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ansible/inventories/{gesis-acceptance => gesis-acceptance} (100%) diff --git a/ansible/inventories/gesis-acceptance b/ansible/inventories/gesis-acceptance similarity index 100% rename from ansible/inventories/gesis-acceptance rename to ansible/inventories/gesis-acceptance From 6e10155628f383137477037324febf1b8bee9ba0 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 17:49:41 +0100 Subject: [PATCH 109/128] Add missing information to Helm deploy --- ansible/roles/harbor/tasks/main.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index 73bdc68a9..220e2b7e3 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -10,7 +10,7 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: "{{ harbor_namespace }}-jobservice" + name: "{{ harbor_name }}-jobservice" labels: harbor: jobservice spec: @@ -39,7 +39,7 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: "{{ harbor_namespace }}-registry" + name: "{{ harbor_name }}-registry" labels: harbor: registry spec: @@ -68,7 +68,7 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: "{{ harbor_namespace }}-registry" + name: "{{ harbor_name }}-registry" labels: harbor: registry spec: @@ -97,7 +97,7 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: "{{ harbor_namespace }}-registry" + name: "{{ harbor_name }}-registry" labels: harbor: registry spec: @@ -126,7 +126,7 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: "{{ harbor_namespace }}-registry" + name: "{{ harbor_name }}-registry" labels: harbor: registry spec: @@ -150,7 +150,8 @@ - name: Deploy harbor kubernetes.core.helm: - name: harbor + chart_ref: harbor/harbor chart_version: 1.16.0 + release_name: "{{ harbor_name }}" release_namespace: "{{ harbor_namespace }}" create_namespace: true From 6685c9230bf36045ea6fb0a4c2a1670f4cb8ddcf Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 17:52:58 +0100 Subject: [PATCH 110/128] Add missing variable --- ansible/roles/harbor/vars/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/harbor/vars/main.yml b/ansible/roles/harbor/vars/main.yml index a238de1ed..d78d46665 100644 --- a/ansible/roles/harbor/vars/main.yml +++ b/ansible/roles/harbor/vars/main.yml @@ -1,3 +1,4 @@ +harbor_name: harbor harbor_namespace: harbor harbor_jobservice_storage: 10Gi harbor_registry_storage: 10Gi From f37f52c4fd5095dbd6e5772cb4c1841d4631dee4 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 18:12:15 +0100 Subject: [PATCH 111/128] Update storage class name --- ansible/roles/harbor/tasks/main.yml | 24 +++++++++++++++++++----- ansible/roles/harbor/vars/main.yml | 1 + 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index 220e2b7e3..ff2ca5800 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -20,7 +20,7 @@ accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Delete - storageClassName: standard + storageClassName: "{{ harbor_storage_class_name }}" local: path: /harbor/jobservice nodeAffinity: @@ -49,7 +49,7 @@ accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Delete - storageClassName: standard + storageClassName: "{{ harbor_storage_class_name }}" local: path: /harbor/registry nodeAffinity: @@ -78,7 +78,7 @@ accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Delete - storageClassName: standard + storageClassName: "{{ harbor_storage_class_name }}" local: path: /harbor/registry nodeAffinity: @@ -107,7 +107,7 @@ accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Delete - storageClassName: standard + storageClassName: "{{ harbor_storage_class_name }}" local: path: /harbor/registry nodeAffinity: @@ -136,7 +136,7 @@ accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Delete - storageClassName: standard + storageClassName: "{{ harbor_storage_class_name }}" local: path: /harbor/registry nodeAffinity: @@ -155,3 +155,17 @@ release_name: "{{ harbor_name }}" release_namespace: "{{ harbor_namespace }}" create_namespace: true + values: + persistence: + persistentVolumeClaim: + registry: + storageClass: "{{ harbor_storage_class_name }}" + jobservice: + jobLog: + storageClass: "{{ harbor_storage_class_name }}" + database: + storageClass: "{{ harbor_storage_class_name }}" + redis: + storageClass: "{{ harbor_storage_class_name }}" + trivy: + storageClass: "{{ harbor_storage_class_name }}" diff --git a/ansible/roles/harbor/vars/main.yml b/ansible/roles/harbor/vars/main.yml index d78d46665..2a44578cc 100644 --- a/ansible/roles/harbor/vars/main.yml +++ b/ansible/roles/harbor/vars/main.yml @@ -1,5 +1,6 @@ harbor_name: harbor harbor_namespace: harbor +harbor_storage_class_name: standard harbor_jobservice_storage: 10Gi harbor_registry_storage: 10Gi harbor_redis_storage: 10Gi From 34eda4ac26807fa569c41d357dee84e476952b06 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 18:18:46 +0100 Subject: [PATCH 112/128] Fix name of storage unit --- ansible/roles/harbor/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index ff2ca5800..1d442d237 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -68,7 +68,7 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: "{{ harbor_name }}-registry" + name: "{{ harbor_name }}-redis" labels: harbor: registry spec: @@ -97,7 +97,7 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: "{{ harbor_name }}-registry" + name: "{{ harbor_name }}-trivy" labels: harbor: registry spec: @@ -126,7 +126,7 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: "{{ harbor_name }}-registry" + name: "{{ harbor_name }}-database" labels: harbor: registry spec: From ac60a4f9b48c3e4a4fde9f437a14bf512fb474e5 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 18:20:31 +0100 Subject: [PATCH 113/128] Use variable for harbor version --- ansible/roles/harbor/tasks/main.yml | 2 +- ansible/roles/harbor/vars/main.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index 1d442d237..8da7a7bda 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -151,7 +151,7 @@ - name: Deploy harbor kubernetes.core.helm: chart_ref: harbor/harbor - chart_version: 1.16.0 + chart_version: "{{ harbor_version }}" release_name: "{{ harbor_name }}" release_namespace: "{{ harbor_namespace }}" create_namespace: true diff --git a/ansible/roles/harbor/vars/main.yml b/ansible/roles/harbor/vars/main.yml index 2a44578cc..1619e94c1 100644 --- a/ansible/roles/harbor/vars/main.yml +++ b/ansible/roles/harbor/vars/main.yml @@ -1,3 +1,4 @@ +harbor_version: 1.16.0 harbor_name: harbor harbor_namespace: harbor harbor_storage_class_name: standard From 38ca8e45a7cbb79d8767e39afeddda5338429b8b Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 18:27:25 +0100 Subject: [PATCH 114/128] Fix metadata for PersistentVolume --- docs/.gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docs/.gitattributes diff --git a/docs/.gitattributes b/docs/.gitattributes new file mode 100644 index 000000000..07fe41c52 --- /dev/null +++ b/docs/.gitattributes @@ -0,0 +1,2 @@ +# GitHub syntax highlighting +pixi.lock linguist-language=YAML linguist-generated=true From f30d28adbfb1c40d748904b0cffbd1a93c98b8e6 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 18:44:40 +0100 Subject: [PATCH 115/128] Fix persistent volume --- ansible/roles/harbor/tasks/main.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index 8da7a7bda..ebcd33b67 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -18,7 +18,7 @@ storage: "{{ harbor_jobservice_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: "{{ harbor_storage_class_name }}" local: @@ -47,7 +47,7 @@ storage: "{{ harbor_registry_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: "{{ harbor_storage_class_name }}" local: @@ -70,17 +70,17 @@ metadata: name: "{{ harbor_name }}-redis" labels: - harbor: registry + harbor: redis spec: capacity: storage: "{{ harbor_redis_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: "{{ harbor_storage_class_name }}" local: - path: /harbor/registry + path: /harbor/redis nodeAffinity: required: nodeSelectorTerms: @@ -99,17 +99,17 @@ metadata: name: "{{ harbor_name }}-trivy" labels: - harbor: registry + harbor: trivy spec: capacity: storage: "{{ harbor_trivy_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: "{{ harbor_storage_class_name }}" local: - path: /harbor/registry + path: /harbor/trivy nodeAffinity: required: nodeSelectorTerms: @@ -128,17 +128,17 @@ metadata: name: "{{ harbor_name }}-database" labels: - harbor: registry + harbor: database spec: capacity: storage: "{{ harbor_database_storage }}" volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: "{{ harbor_storage_class_name }}" local: - path: /harbor/registry + path: /harbor/database nodeAffinity: required: nodeSelectorTerms: From 7bbbc3023a7eaa75efe1e7f15d9f1391d9c8ad8d Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 18:44:59 +0100 Subject: [PATCH 116/128] Update values for Harbor Helm chart --- ansible/roles/harbor/tasks/main.yml | 1 + ansible/vault/gesis-acceptance.yml | 45 +++++++++++++++-------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index ebcd33b67..dfb8dfffc 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -156,6 +156,7 @@ release_namespace: "{{ harbor_namespace }}" create_namespace: true values: + harborAdminPassword: "{{ HARBOR_ADMIN_PASSWORD }}" persistence: persistentVolumeClaim: registry: diff --git a/ansible/vault/gesis-acceptance.yml b/ansible/vault/gesis-acceptance.yml index c6ce4e92c..b582c8376 100644 --- a/ansible/vault/gesis-acceptance.yml +++ b/ansible/vault/gesis-acceptance.yml @@ -1,22 +1,25 @@ $ANSIBLE_VAULT;1.1;AES256 -65666231316164316637653330376337383937373938613334343066376139326661643962376237 -3739366536353237356539656138383164326139333139390a333134313565323232646639313162 -61656433306461343266393566626465316239353933303136633034343231666337363838623563 -6633633234626132390a333632353730353066326438623663383634343532333539366363333334 -34646163313065393732306363353231633239313637646339623032626366626436346234376130 -66636432383138383838616434303931316334386665303563376336623930356638666366333561 -66353830353361343335623737653130383862353638393336303866323738303865623934303830 -66663164353837626636653766646233666164393564396233656665646636643862643035383733 -65376535346438623032316666333265643135653035373139626232646430623733383134656533 -34323737613565663536643430613832636666653030383066316632336363323734326339376162 -39343665393661623530303236353165656130396137373634363265346362623832653563613338 -31313261646333656362636134306162666133373334653933366531643063643537353663353932 -39386538626664393536363035646265643832303961323636653037356433346266353963666164 -32653334653936633130316463303061343938363630376663613639636338343331353732363837 -37616137373834333836393137333131643432653239313432623462616537353337303432393736 -34333463636566373330346437653037313366633762623161616564376639376561333561366530 -37356235373336303563373137393263626532356333666166396435346565333964316263393665 -32636239396563326635363636396435623731613364376632336261643064336530616235386631 -37336230323331323838326331303831616337363833616563306131393733666663303836636366 -38656336373763353836643536376239316463353862323332626661346366636236613530366464 -36363832656263633161303335613332396237353865643964626462653565386562 +33363162666433643064343861356331353636383861613134396366653734346162663739316437 +3036656631303962643932353761343133316231313138330a396438633163613839323534376262 +65346530316334633337646632343161343538393566626164353332353733363135376363343836 +6634333333346638320a646630313964303832616531383830373738383734646536383630613062 +34333138303366353366633465316435363435373466366532363339356135653231366163336165 +37366536633934316433656532633837636635303838656339643638383263656461303161643037 +37313133376231613963613938313866353231343833383161356333333035613231623838616461 +36376563353138636632326638626239646564313363623565396564303066336564656235343935 +65643634646138663134316439653065656662363635666466333937323337366564653937616136 +33623963386336353231386135623866616632366365663036646439623331666535306362353562 +34613439666232616138393237323635363432353165346665383037303032636235643735356164 +32643363613137356337396563366534353139343237396539616365656439323966326434653864 +64656662646234363734626233313235643338356336333038323134393439626564656538613363 +38633735616437396364663565316263303438336636313539623630633066633636326333666530 +33663330343036613566346164396636323965666132663361343133666138623938383831613934 +32386562656531393162626435333639643232326265666562346537346334313961393761366234 +37396231343665363533356339636435313934636464353130386566376330356461613531306237 +30333339343637663962303264653732346562303131616364386565383262343936313162393766 +66333333653339633766316536356661333062656162316131346436613062333132623332363537 +63653736363362363562646265383537613366643535346665316432336266373132306463376436 +65656438353361373132393766356639616435623766333437636635383263313938633935393961 +30323963336563366135316565613137383136333964366437636633346237366130393537616662 +36313965623130666237393337366431616635653235623961623432646664643032323038393430 +62316439383963643964 From cf7dda7e85f314de0a9ea812849b7e19fc05fd48 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 09:13:35 +0100 Subject: [PATCH 117/128] Fix typo in GitLab CI --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0d3744cb..1a78db752 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ stages: when: manual allow_failure: true -.geis-merge-request: +.gesis-merge-request: rules: - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "merge_request_event" changes: @@ -29,7 +29,7 @@ stages: - config/**/* - secrets/**/* -.geis-push-main: +.gesis-push-main: rules: - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'main' @@ -104,8 +104,8 @@ gesis helm acceptance deploy: stage: deploy-acceptance-helm rules: - !reference [.gesis-manual-web, rules] - - !reference [.geis-merge-request, rules] - - !reference [.geis-push-main, rules] + - !reference [.gesis-merge-request, rules] + - !reference [.gesis-push-main, rules] variables: HELM_ENVIRONMENT: acceptance extends: @@ -116,7 +116,7 @@ gesis helm production deploy: stage: deploy-production-helm rules: - !reference [.gesis-manual-web, rules] - - !reference [.geis-push-main, rules] + - !reference [.gesis-push-main, rules] variables: HELM_ENVIRONMENT: stage extends: @@ -126,7 +126,7 @@ smoke test to acceptance cluster: stage: test-acceptance rules: - !reference [.gesis-manual-web, rules] - - !reference [.geis-merge-request, rules] - - !reference [.geis-push-main, rules] + - !reference [.gesis-merge-request, rules] + - !reference [.gesis-push-main, rules] script: - curl https://notebooks-test.gesis.org/binder/ From 9850e9a303764f7da5e352dc4f91423df46905d4 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 09:22:14 +0100 Subject: [PATCH 118/128] Update GitLab CI rules --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1a78db752..e5af3e89c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,7 +48,8 @@ include: ssh-user: ansible ssh-key-type: ed25519 rules: - - if: $CI_SERVER_HOST == "git.gesis.org" + - if: $CI_SERVER_HOST == "git.gesis.org" && $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'main' .gesis helm deploy: image: From 60b42696c8044b7a8a42b4711c546b907d6715bf Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 09:34:30 +0100 Subject: [PATCH 119/128] Improve GitLab CI --- .gitlab-ci.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e5af3e89c..ed5a35393 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,10 @@ stages: rules: - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'main' +.gesis-push-gesis: + rules: + - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'gesis' + include: - component: $CI_SERVER_FQDN/rse/docker/images/ansible/ansible-lint@10.2.6 inputs: @@ -50,6 +54,7 @@ include: rules: - if: $CI_SERVER_HOST == "git.gesis.org" && $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'main' + - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'gesis' .gesis helm deploy: image: @@ -107,27 +112,37 @@ gesis helm acceptance deploy: - !reference [.gesis-manual-web, rules] - !reference [.gesis-merge-request, rules] - !reference [.gesis-push-main, rules] + - !reference [.gesis-push-gesis, rules] variables: HELM_ENVIRONMENT: acceptance extends: - .gesis helm deploy -gesis helm production deploy: +.gesis helm production deploy: resource_group: production stage: deploy-production-helm rules: - !reference [.gesis-manual-web, rules] - !reference [.gesis-push-main, rules] + - !reference [.gesis-push-gesis, rules] variables: HELM_ENVIRONMENT: stage extends: - .gesis helm deploy +.smoke test: + stage: test-acceptance + variables: + INTERACTIVE_URL: url + script: + - curl $INTERACTIVE_URL + smoke test to acceptance cluster: stage: test-acceptance rules: - !reference [.gesis-manual-web, rules] - !reference [.gesis-merge-request, rules] - !reference [.gesis-push-main, rules] - script: - - curl https://notebooks-test.gesis.org/binder/ + - !reference [.gesis-push-gesis, rules] + variables: + INTERACTIVE_URL: https://notebooks-test.gesis.org/binder/ From 306c7b3b22caf864d8931e3d2f61b5b42c236d9c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 09:54:03 +0100 Subject: [PATCH 120/128] Fix GitLab CI Add missing extends. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed5a35393..0f9b3fd11 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -146,3 +146,5 @@ smoke test to acceptance cluster: - !reference [.gesis-push-gesis, rules] variables: INTERACTIVE_URL: https://notebooks-test.gesis.org/binder/ + extends: + - .smoke test \ No newline at end of file From 1f15674278b07415d8fa9767b685d3a8612ca25c Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Thu, 2 Jan 2025 19:12:31 +0100 Subject: [PATCH 121/128] Change type of expose for Harbor --- ansible/roles/harbor/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index dfb8dfffc..711b0dcd2 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -157,6 +157,12 @@ create_namespace: true values: harborAdminPassword: "{{ HARBOR_ADMIN_PASSWORD }}" + expose: + type: loadBalancer + tls: + enabled: false + loadBalancer: + name: "{{ harbor_name }}" persistence: persistentVolumeClaim: registry: From 34094e015e4c9560db24660505a05e02fa11b064 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 14:22:19 +0100 Subject: [PATCH 122/128] Update value of Ansible K8S_INGRESS --- ansible/inventories/gesis-acceptance | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/inventories/gesis-acceptance b/ansible/inventories/gesis-acceptance index d284d2397..2b146d49c 100644 --- a/ansible/inventories/gesis-acceptance +++ b/ansible/inventories/gesis-acceptance @@ -10,7 +10,7 @@ K8S_CONTROL_PLANE_ENDPOINT=194.95.75.21 K8S_CONTROL_PLANE_ALIAS=svko-k8s-test01 ; Replace this variable with a filter ; This must match the group ingress -K8S_INGRESS=194.95.75.22 +K8S_INGRESS=194.95.75.20 [notebooks_gesis_org] svko-css-backup-node From 4f516f7b2a986dd833a305b52cdee3f3b4e0284b Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 14:44:16 +0100 Subject: [PATCH 123/128] Change Harbor expose type to ClusterIP Read https://kubernetes.github.io/ingress-nginx/user-guide/basic-usage/ --- ansible/roles/harbor/tasks/main.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index 711b0dcd2..93c5a3665 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -158,11 +158,9 @@ values: harborAdminPassword: "{{ HARBOR_ADMIN_PASSWORD }}" expose: - type: loadBalancer + type: ClusterIP tls: enabled: false - loadBalancer: - name: "{{ harbor_name }}" persistence: persistentVolumeClaim: registry: From 67c29df316e728bec317a0b96c705bbe7335db8e Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 14:45:23 +0100 Subject: [PATCH 124/128] Configure Ingress NGINX Read https://github.com/kubernetes/ingress-nginx --- .../k8s-control-panel/tasks/ingress-nginx.yml | 20 +++++++++++++++++++ .../roles/k8s-control-panel/tasks/main.yml | 3 +++ .../roles/k8s-control-panel/tasks/metallb.yml | 1 + ansible/roles/k8s-control-panel/vars/main.yml | 2 ++ 4 files changed, 26 insertions(+) create mode 100644 ansible/roles/k8s-control-panel/tasks/ingress-nginx.yml diff --git a/ansible/roles/k8s-control-panel/tasks/ingress-nginx.yml b/ansible/roles/k8s-control-panel/tasks/ingress-nginx.yml new file mode 100644 index 000000000..25b4ce769 --- /dev/null +++ b/ansible/roles/k8s-control-panel/tasks/ingress-nginx.yml @@ -0,0 +1,20 @@ +- name: Add a Ingress NGINX Controller Helm repository + kubernetes.core.helm_repository: + repo_name: ingress-nginx + repo_url: https://kubernetes.github.io/ingress-nginx +- name: Create Ingress NGINX Controller Kubernetes namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: ingress-nginx +- name: Deploy Ingress NGINX Controller + kubernetes.core.helm: + release_name: ingress-nginx + release_namespace: ingress-nginx + chart_ref: ingress-nginx/ingress-nginx + chart_version: '{{ k8s_control_panel_ingress_nginx_version }}' + create_namespace: false + history_max: 3 diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 2143c6d49..19bb55674 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -167,3 +167,6 @@ - name: Add MetalLB to Kubernetes cluster ansible.builtin.import_tasks: file: metallb.yml +- name: Add Ingress NGINX Controller to Kubernetes cluster + ansible.builtin.import_tasks: + file: ingress-nginx.yml diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index 9088a9f27..3d12dac2c 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -21,6 +21,7 @@ release_name: metallb release_namespace: metallb chart_ref: metallb/metallb + chart_version: '{{ k8s_control_panel_metallb_version }}' create_namespace: false history_max: 3 - name: Create MetalLB Kubernetes IP Address Pool diff --git a/ansible/roles/k8s-control-panel/vars/main.yml b/ansible/roles/k8s-control-panel/vars/main.yml index 3474a6307..b22109826 100644 --- a/ansible/roles/k8s-control-panel/vars/main.yml +++ b/ansible/roles/k8s-control-panel/vars/main.yml @@ -1,3 +1,5 @@ k8s_control_panel_calico_version: "3.28.2" k8s_control_panel_cidr: "10.244.0.0/16" +k8s_control_panel_metallb_version: 0.14.9 k8s_control_panel_metallb_ip_address_pool_name: "gesis" +k8s_control_panel_ingress_nginx_version: 4.12.0 From 00446b221d97bc148e4d520be108b864b7238c9d Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 15:00:19 +0100 Subject: [PATCH 125/128] Fix Kubernetes configuration file is group-readable --- ansible/roles/k8s-control-panel/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/k8s-control-panel/tasks/main.yml b/ansible/roles/k8s-control-panel/tasks/main.yml index 19bb55674..ce4fc1fba 100644 --- a/ansible/roles/k8s-control-panel/tasks/main.yml +++ b/ansible/roles/k8s-control-panel/tasks/main.yml @@ -38,7 +38,7 @@ ansible.builtin.file: path: /home/ansible/.kube state: directory - mode: u=rwx,g=rx,o=rx + mode: u=rwx,g=x,o=x owner: ansible group: ansible - name: Copies admin.conf to user's kube config @@ -48,7 +48,7 @@ remote_src: true owner: ansible group: ansible - mode: u=rw,g=r,o= + mode: u=rw,g=,o= - name: Get the token for joining the worker nodes ansible.builtin.shell: > kubeadm token create --print-join-command From 2d909276329352e026c5571c7711b6575bb2e10f Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 15:07:54 +0100 Subject: [PATCH 126/128] Force update Helm repository --- ansible/roles/k8s-control-panel/tasks/ingress-nginx.yml | 1 + ansible/roles/k8s-control-panel/tasks/metallb.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/ansible/roles/k8s-control-panel/tasks/ingress-nginx.yml b/ansible/roles/k8s-control-panel/tasks/ingress-nginx.yml index 25b4ce769..8d0cb940c 100644 --- a/ansible/roles/k8s-control-panel/tasks/ingress-nginx.yml +++ b/ansible/roles/k8s-control-panel/tasks/ingress-nginx.yml @@ -2,6 +2,7 @@ kubernetes.core.helm_repository: repo_name: ingress-nginx repo_url: https://kubernetes.github.io/ingress-nginx + force_update: true - name: Create Ingress NGINX Controller Kubernetes namespace kubernetes.core.k8s: state: present diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index 3d12dac2c..1f0bbd8e7 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -2,6 +2,7 @@ kubernetes.core.helm_repository: repo_name: metallb repo_url: https://metallb.github.io/metallb + force_update: true - name: Create MetalLB Kubernetes namespace kubernetes.core.k8s: state: present From a2014075253a48badcc01d02e336aa590338d15d Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 15:54:27 +0100 Subject: [PATCH 127/128] Configure Ingress for Harbor --- ansible/gesis.yml | 5 ++++- ansible/inventories/gesis-acceptance | 2 ++ ansible/roles/harbor/tasks/main.yml | 25 +++++++++++++++++++++++++ ansible/roles/harbor/vars/main.yml | 2 ++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ansible/gesis.yml b/ansible/gesis.yml index afdee1ac4..bfe6774cc 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -26,7 +26,10 @@ hosts: kubernetes_control_panel gather_facts: false roles: - - harbor + - role: harbor + vars: + harbor_domain: '{{ HARBOR_DOMAIN }}' + harbor_path: '{{ HARBOR_PATH }}' - name: Configure JupyterHub workers hosts: jupyterhub_single_user gather_facts: false diff --git a/ansible/inventories/gesis-acceptance b/ansible/inventories/gesis-acceptance index 2b146d49c..3697dab3f 100644 --- a/ansible/inventories/gesis-acceptance +++ b/ansible/inventories/gesis-acceptance @@ -22,6 +22,8 @@ svko-k8s-test01 GRAFANA_CAPACITY_STORAGE=2Gi JUPYTERHUB_CAPACITY_STORAGE=2Gi PROMETHEUS_CAPACITY_STORAGE=15Gi +HARBOR_DOMAIN=notebooks.gesis.org +HARBOR_PATH='/' [kubernetes_workers] svko-k8s-test02 diff --git a/ansible/roles/harbor/tasks/main.yml b/ansible/roles/harbor/tasks/main.yml index 93c5a3665..91a7b1fdd 100644 --- a/ansible/roles/harbor/tasks/main.yml +++ b/ansible/roles/harbor/tasks/main.yml @@ -155,6 +155,7 @@ release_name: "{{ harbor_name }}" release_namespace: "{{ harbor_namespace }}" create_namespace: true + history_max: 3 values: harborAdminPassword: "{{ HARBOR_ADMIN_PASSWORD }}" expose: @@ -174,3 +175,27 @@ storageClass: "{{ harbor_storage_class_name }}" trivy: storageClass: "{{ harbor_storage_class_name }}" + +# Based on https://kubernetes.github.io/ingress-nginx/user-guide/basic-usage/ +- name: Create a ingress resources + kubernetes.core.k8s: + state: present + definition: + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: 'ingress-{{ harbor_name }}' + spec: + rules: + - host: '{{ harbor_domain }}' + http: + paths: + - path: '{{ harbor_path }}' + pathType: Prefix + backend: + service: + name: '{{ harbor_name }}' + port: + number: 80 + # Don't change the ingressClassName + ingressClassName: nginx diff --git a/ansible/roles/harbor/vars/main.yml b/ansible/roles/harbor/vars/main.yml index 1619e94c1..a7c50e301 100644 --- a/ansible/roles/harbor/vars/main.yml +++ b/ansible/roles/harbor/vars/main.yml @@ -7,3 +7,5 @@ harbor_registry_storage: 10Gi harbor_redis_storage: 10Gi harbor_trivy_storage: 10Gi harbor_database_storage: 10Gi +harbor_domain: harbor.localhost +harbor_path: '/' From 60d4e61152b62c71e25aeed37882d3cca00ec179 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 3 Jan 2025 16:02:15 +0100 Subject: [PATCH 128/128] Improve use of Ansible roles variables --- ansible/gesis.yml | 5 ++++- ansible/roles/k8s-control-panel/tasks/metallb.yml | 2 +- ansible/roles/k8s-control-panel/vars/main.yml | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ansible/gesis.yml b/ansible/gesis.yml index bfe6774cc..b26cdb02d 100644 --- a/ansible/gesis.yml +++ b/ansible/gesis.yml @@ -9,7 +9,10 @@ gather_facts: false become: true roles: - - k8s-control-panel + - role: k8s-control-panel + vars: + k8s_control_panel_addresses_begin: '{{ K8S_INGRESS }}' + k8s_control_panel_addresses_end: '{{ K8S_INGRESS }}' - name: Configure Kubernetes workers hosts: kubernetes_workers gather_facts: false diff --git a/ansible/roles/k8s-control-panel/tasks/metallb.yml b/ansible/roles/k8s-control-panel/tasks/metallb.yml index 1f0bbd8e7..b7b496cb8 100644 --- a/ansible/roles/k8s-control-panel/tasks/metallb.yml +++ b/ansible/roles/k8s-control-panel/tasks/metallb.yml @@ -37,7 +37,7 @@ spec: addresses: # TODO Use Jinja filter to automate this. - - "{{ K8S_INGRESS }}-{{ K8S_INGRESS }}" + - "{{ k8s_control_panel_addresses_begin }}-{{ k8s_control_panel_addresses_end }}" - name: Configure L2 Advertisement for MetalLB kubernetes.core.k8s: state: present diff --git a/ansible/roles/k8s-control-panel/vars/main.yml b/ansible/roles/k8s-control-panel/vars/main.yml index b22109826..31cfa853d 100644 --- a/ansible/roles/k8s-control-panel/vars/main.yml +++ b/ansible/roles/k8s-control-panel/vars/main.yml @@ -3,3 +3,5 @@ k8s_control_panel_cidr: "10.244.0.0/16" k8s_control_panel_metallb_version: 0.14.9 k8s_control_panel_metallb_ip_address_pool_name: "gesis" k8s_control_panel_ingress_nginx_version: 4.12.0 +k8s_control_panel_addresses_begin: 0.0.0.0 +k8s_control_panel_addresses_end: 0.0.0.0