Streamline deployment of GESIS stage server

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[ansible/**]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true

.gitattributes

@@ -1 +1,3 @@
 **/secrets/** filter=git-crypt diff=git-crypt
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML linguist-generated=true

.gitignore

@@ -19,3 +19,6 @@ travis/crypt-key
 env
 
 .terraform
+# pixi environments
+.pixi
+*.egg-info

.gitlab-ci.yml

@@ -0,0 +1,150 @@
+variables:
+  GIT_STRATEGY: clone
+  GIT_CLEAN_FLAGS: "-ffdx"
+
+stages:
+  - lint
+  - deploy-acceptance-ansible
+  - deploy-acceptance-helm
+  - test-acceptance
+  - deploy-production-nginx
+  - deploy-production-helm
+
+.gesis-manual-web:
+  rules:
+    - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == 'web'
+      when: manual
+      allow_failure: true
+
+.gesis-merge-request:
+  rules:
+    - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - .gitlab.yml
+      when: manual
+    - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - ansible/**/*
+        - mybinder/**/*
+        - config/**/*
+        - secrets/**/*
+
+.gesis-push-main:
+  rules:
+    - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'main'
+
+.gesis-push-gesis:
+  rules:
+    - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'gesis'
+
+include:
+  - component: $CI_SERVER_FQDN/rse/docker/images/ansible/[email protected]
+    inputs:
+      stage: lint
+      dir: ansible
+
+  - component: $CI_SERVER_FQDN/rse/docker/images/ansible/[email protected]
+    inputs:
+      stage: deploy-acceptance-ansible
+      dir: ansible
+      inventory: gesis-acceptance
+      playbook: gesis.yml
+      ssh-user: ansible
+      ssh-key-type: ed25519
+    rules:
+      - if: $CI_SERVER_HOST == "git.gesis.org" && $CI_PIPELINE_SOURCE == "merge_request_event"
+      - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'main'
+      - if: $CI_SERVER_HOST == 'git.gesis.org' && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == 'gesis'
+
+.gesis helm deploy:
+  image:
+    name: docker-private.gesis.intra/gesis/ilcm/orc2/k8s:latest
+    entrypoint: [""]
+  rules:
+      - if: $CI_SERVER_HOST == "git.gesis.org"
+  variables:
+    HELM_ENVIRONMENT: template
+  script:
+    - cat $git_crypt_secret_key | base64 -d > git_crypt_secret_key
+    - git-crypt unlock git_crypt_secret_key
+    - kubectl config use-context ${CI_PROJECT_PATH}:${HELM_ENVIRONMENT}
+    - helm version
+    - |
+      for d in ./mybinder*/; do
+        helm dependency update "$d";
+      done
+    # - |
+    #   for chart in mybinder-kube-system mybinder-tigera-operator; do
+    #     helm upgrade \
+    #     ${chart:9} ./${chart} \
+    #     --cleanup-on-fail \
+    #     --create-namespace \
+    #     --history-max 3 \
+    #     --install \
+    #     --namespace=${chart};
+    #   done
+    - |
+      helm lint ./mybinder \
+      --values ./config/gesis-${HELM_ENVIRONMENT}.yaml \
+      --values ./secrets/config/common/common.yaml \
+      --values ./secrets/config/common/cryptnono.yaml \
+      --values ./secrets/config/common/gesis.yaml \
+      --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml
+    - |
+      helm upgrade \
+      binderhub ./mybinder \
+      --cleanup-on-fail \
+      --create-namespace \
+      --history-max 3 \
+      --install \
+      --namespace=gesis \
+      --render-subchart-notes \
+      --values ./config/gesis-${HELM_ENVIRONMENT}.yaml \
+      --values ./secrets/config/common/common.yaml \
+      --values ./secrets/config/common/cryptnono.yaml \
+      --values ./secrets/config/common/gesis.yaml \
+      --values ./secrets/config/gesis-${HELM_ENVIRONMENT}.yaml
+
+gesis helm acceptance deploy:
+  resource_group: acceptance
+  stage: deploy-acceptance-helm
+  rules:
+    - !reference [.gesis-manual-web, rules]
+    - !reference [.gesis-merge-request, rules]
+    - !reference [.gesis-push-main, rules]
+    - !reference [.gesis-push-gesis, rules]
+  variables:
+    HELM_ENVIRONMENT: acceptance
+  extends:
+    - .gesis helm deploy
+
+.gesis helm production deploy:
+  resource_group: production
+  stage: deploy-production-helm
+  rules:
+    - !reference [.gesis-manual-web, rules]
+    - !reference [.gesis-push-main, rules]
+    - !reference [.gesis-push-gesis, rules]
+  variables:
+    HELM_ENVIRONMENT: stage
+  extends:
+    - .gesis helm deploy
+
+.smoke test:
+  stage: test-acceptance
+  variables:
+    INTERACTIVE_URL: url
+  script:
+    - curl $INTERACTIVE_URL
+
+smoke test to acceptance cluster:
+  stage: test-acceptance
+  rules:
+    - !reference [.gesis-manual-web, rules]
+    - !reference [.gesis-merge-request, rules]
+    - !reference [.gesis-push-main, rules]
+    - !reference [.gesis-push-gesis, rules]
+  variables:
+    INTERACTIVE_URL: https://notebooks-test.gesis.org/binder/
+  extends:
+    - .smoke test

.gitlab/agents/stage/config.yaml

@@ -0,0 +1,3 @@
+ci_access:
+  projects:
+    - id: methods-hub/interactive-environment

ansible/gesis.yml

@@ -0,0 +1,46 @@
+- name: Configure servers that are part of Kubernetes cluster
+  hosts: all
+  gather_facts: false
+  become: true
+  roles:
+    - k8s-common
+- name: Configure Kubernetes control panel
+  hosts: kubernetes_control_panel
+  gather_facts: false
+  become: true
+  roles:
+    - role: k8s-control-panel
+      vars:
+        k8s_control_panel_addresses_begin: '{{ K8S_INGRESS }}'
+        k8s_control_panel_addresses_end: '{{ K8S_INGRESS }}'
+- name: Configure Kubernetes workers
+  hosts: kubernetes_workers
+  gather_facts: false
+  become: true
+  roles:
+    - k8s-worker
+- name: Configure Kubernetes Persistent Volumes
+  hosts: kubernetes_control_panel
+  gather_facts: false
+  become: true
+  roles:
+    - k8s-pv
+- name: Configure Harbor
+  hosts: kubernetes_control_panel
+  gather_facts: false
+  roles:
+    - role: harbor
+      vars:
+        harbor_domain: '{{ HARBOR_DOMAIN }}'
+        harbor_path: '{{ HARBOR_PATH }}'
+- name: Configure JupyterHub workers
+  hosts: jupyterhub_single_user
+  gather_facts: false
+  become: true
+  roles:
+    - k8s-worker
+- name: Configure mybinder.org Kubernetes cluster
+  hosts: kubernetes_control_panel
+  gather_facts: false
+  roles:
+    - mybinder

ansible/inventories/gesis-acceptance

@@ -0,0 +1,51 @@
+[all]
+svko-css-backup-node ansible_host=194.95.75.20 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_20 }}'
+svko-k8s-test01 ansible_host=194.95.75.21 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_21 }}'
+svko-k8s-test02 ansible_host=194.95.75.22 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_22 }}'
+svko-k8s-test03 ansible_host=194.95.75.23 ansible_ssh_user=ansible ansible_become_pass='{{ become_pass_194_95_75_23 }}'
+
+[all:vars]
+INVENTORY_NAME=stage
+K8S_CONTROL_PLANE_ENDPOINT=194.95.75.21
+K8S_CONTROL_PLANE_ALIAS=svko-k8s-test01
+; Replace this variable with a filter
+; This must match the group ingress
+K8S_INGRESS=194.95.75.20
+
+[notebooks_gesis_org]
+svko-css-backup-node
+
+[kubernetes_control_panel]
+svko-k8s-test01
+
+[kubernetes_control_panel:vars]
+GRAFANA_CAPACITY_STORAGE=2Gi
+JUPYTERHUB_CAPACITY_STORAGE=2Gi
+PROMETHEUS_CAPACITY_STORAGE=15Gi
+HARBOR_DOMAIN=notebooks.gesis.org
+HARBOR_PATH='/'
+
+[kubernetes_workers]
+svko-k8s-test02
+svko-k8s-test03
+
+[ingress]
+svko-css-backup-node
+
+[harbor]
+svko-k8s-test03
+
+[binderhub]
+svko-k8s-test02
+
+[jupyterhub]
+svko-k8s-test02
+
+[jupyterhub_single_user]
+svko-k8s-test02
+
+[prometheus]
+svko-css-backup-node
+
+[grafana]
+svko-css-backup-node