feat: k3s ansible playbook (#1071)

It changes the directory layout of the ansible playbook to a
more "best practices" friendly approach using ansible roles and
a real inventory file

Co-authored-by: Ilya Kreymer <[email protected]>

ansible/inventory/sample-k3s/group_vars/all.yml

@@ -0,0 +1,10 @@
+---
+k3s_version: v1.22.3+k3s1
+ansible_user: debian
+systemd_dir: /etc/systemd/system
+controller_ip: "{{ hostvars[groups['controller'][0]]['ansible_host'] | default(groups['controller'][0]) }}"
+extra_server_args: "--disable traefik"
+extra_agent_args: ""
+project_name: browsertrix-cloud
+domain: my-domain.example.com
+email: [email protected]

ansible/inventory/sample-k3s/hosts.ini

@@ -0,0 +1,12 @@
+[controller]
+# Set to the IP address of the k3s host node
+127.0.0.1
+
+# Uncomment for multi-node deployment
+# [node]
+# 192.168.1.2
+
+[k3s_cluster:children]
+controller
+# Uncomment for multi-node deployment
+# node

ansible/playbooks/install_k3s.yml

@@ -0,0 +1,30 @@
+---
+
+# Can be skipped if k3s is installed, this installs k3s
+- hosts: k3s_cluster
+  gather_facts: yes
+  connection: local # Comment if deploying to remote host
+  become: yes
+  roles:
+    - role: prereq
+    - role: download
+
+# Can be skipped if k3s is installed, this configures the master k3s node
+- hosts: controller
+  connection: local # Comment if deploying to remote host
+  become: yes
+  roles:
+    - role: k3s/master
+
+# Uncomment for multi-node deployment
+# - hosts: node
+#   roles:
+#     - role: k3s/node
+
+# Ansible controller to install browsertrix cloud
+- hosts: 127.0.0.1
+  connection: local
+  become: yes # Can be removed if not using the btrix/prereq role
+  roles:
+    - role: btrix/prereq # Only required if you wish to install & configure Helm / Kubectl
+    - role: btrix/install

ansible/roles/btrix/install/tasks/main.yml

@@ -0,0 +1,55 @@
+---
+- name: Create directory .kube
+  file:
+    path: ~{{ ansible_user }}/.kube
+    state: directory
+    owner: "{{ ansible_user }}"
+    mode: "u=rwx,g=rx,o="
+
+- name: Check whether kube config exists
+  stat:
+    path: ~/.kube/config
+  register: kubeconfig_result
+
+- name: Get k3s config
+  ansible.posix.synchronize:
+    src: rsync://{{ controller_ip }}/home/{{ ansible_user }}/.kube/config
+    dest: ~/.kube/config
+  when: not kubeconfig_result.stat.exists
+
+- name: Check whether CRDs installed
+  ansible.builtin.command: kubectl get crd
+  register: crd_register
+
+- name: Run all admin bookstrap scripts
+  ansible.builtin.command: >-
+      ../chart/admin/logging/scripts/eck_install.sh
+  register: addons_init
+  when: "crd_register.stdout | length < 16"
+  changed_when: true
+
+- name: Install Cert-Manager
+  ansible.builtin.command: kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml
+  changed_when: true
+
+- name: Install ingress-nginx
+  ansible.builtin.command: helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace
+
+- name: Install Metrics Server
+  ansible.builtin.shell: |
+    helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/
+    helm upgrade --install metrics-server metrics-server/metrics-server
+
+- name: Output values yaml
+  ansible.builtin.template:
+    src: k8s-manifest.yaml.j2
+    dest: ../chart/{{ project_name }}-values.yaml
+    mode: u+rw
+
+- name: d_ocean | helm | deploy btrix
+  ansible.builtin.command: helm upgrade --install -f ../chart/values.yaml -f ../chart/{{ project_name }}-values.yaml btrix ../chart/
+  register: helm_result
+  changed_when: helm_result.rc == 0
+  environment:
+    KUBECONFIG: "/home/{{ ansible_user }}/.kube/config"
+  tags: helm_upgrade

ansible/roles/btrix/prereq/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: Gather installed helm version, if there is any
+  ansible.builtin.shell: helm version
+  register: helm_result
+  failed_when: helm_result.rc != 0 and helm_result.rc != 127
+  # Since this is a reporting task, it should never change
+  # as well run and register a result in any case
+  changed_when: false
+  check_mode: false
+
+- name: Install Helm
+  ansible.builtin.shell: |
+    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+    chmod +700 get_helm.sh
+    ./get_helm.sh
+  when: helm_result.rc != 0
+
+- name: Install kubectl
+  ansible.builtin.shell: |
+    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+    install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+- name: Install jq
+  ansible.builtin.package:
+    name: jq
+    state: present

ansible/roles/btrix/templates/k8s-manifest.yaml.j2

@@ -0,0 +1,11 @@
+ingress_class: "nginx"
+
+mongo_auth:
+  username: root
+  password: example
+
+ingress:
+  host: "{{ domain }}"
+  cert_email: "{{ email }}"
+  scheme: "https"
+  tls: true

ansible/roles/download/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+
+- name: Download k3s binary x64
+  get_url:
+    url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s
+    checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt
+    dest: /usr/local/bin/k3s
+    owner: root
+    group: root
+    mode: 0755
+  when: ansible_facts.architecture == "x86_64"
+
+- name: Download k3s binary arm64
+  get_url:
+    url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-arm64
+    checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm64.txt
+    dest: /usr/local/bin/k3s
+    owner: root
+    group: root
+    mode: 0755
+  when:
+    - ( ansible_facts.architecture is search("arm") and
+        ansible_facts.userspace_bits == "64" ) or
+      ansible_facts.architecture is search("aarch64")
+
+- name: Download k3s binary armhf
+  get_url:
+    url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-armhf
+    checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm.txt
+    dest: /usr/local/bin/k3s
+    owner: root
+    group: root
+    mode: 0755
+  when:
+    - ansible_facts.architecture is search("arm")
+    - ansible_facts.userspace_bits == "32"

ansible/roles/k3s/master/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+k3s_server_location: /var/lib/rancher/k3s

ansible/roles/k3s/master/tasks/main.yml

@@ -0,0 +1,91 @@
+---
+
+- name: Copy K3s service file
+  register: k3s_service
+  template:
+    src: "k3s.service.j2"
+    dest: "{{ systemd_dir }}/k3s.service"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Enable and check K3s service
+  systemd:
+    name: k3s
+    daemon_reload: yes
+    state: started
+    enabled: yes
+
+- name: Wait for node-token
+  wait_for:
+    path: "{{ k3s_server_location }}/server/node-token"
+
+- name: Register node-token file access mode
+  stat:
+    path: "{{ k3s_server_location }}/server/node-token"
+  register: p
+
+- name: Change file access node-token
+  file:
+    path: "{{ k3s_server_location }}/server/node-token"
+    mode: "g+rx,o+rx"
+
+- name: Read node-token from master
+  slurp:
+    path: "{{ k3s_server_location }}/server/node-token"
+  register: node_token
+
+- name: Store Master node-token
+  set_fact:
+    token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}"
+
+- name: Restore node-token file access
+  file:
+    path: "{{ k3s_server_location }}/server/node-token"
+    mode: "{{ p.stat.mode }}"
+
+- name: Create directory .kube
+  file:
+    path: ~{{ ansible_user }}/.kube
+    state: directory
+    owner: "{{ ansible_user }}"
+    mode: "u=rwx,g=rx,o="
+
+- name: Copy config file to user home directory
+  copy:
+    src: /etc/rancher/k3s/k3s.yaml
+    dest: ~{{ ansible_user }}/.kube/config
+    remote_src: yes
+    owner: "{{ ansible_user }}"
+    mode: "u=rw,g=,o="
+
+- name: Replace https://localhost:6443 by https://controller-ip:6443
+  command: >-
+    k3s kubectl config set-cluster default
+      --server=https://{{ controller_ip }}:6443
+      --kubeconfig ~{{ ansible_user }}/.kube/config
+  changed_when: true
+
+- name: Check that the kubectl binary exists
+  stat:
+    path: /usr/local/bin/kubectl
+  register: kubectl_result
+
+- name: Check that the crictl binary exists
+  stat:
+    path: /usr/local/bin/crictl
+  register: crictl_result
+
+- name: Create kubectl symlink
+  file:
+    src: /usr/local/bin/k3s
+    dest: /usr/local/bin/kubectl
+    state: link
+  when: not kubectl_result.stat.exists
+
+- name: Create crictl symlink
+  file:
+    src: /usr/local/bin/k3s
+    dest: /usr/local/bin/crictl
+    state: link
+  when: not crictl_result.stat.exists

ansible/roles/k3s/master/templates/k3s.service.j2

@@ -0,0 +1,24 @@
+[Unit]
+Description=Lightweight Kubernetes
+Documentation=https://k3s.io
+After=network-online.target
+
+[Service]
+Type=notify
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args | default("") }}
+KillMode=process
+Delegate=yes
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/k3s/node/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+
+- name: Copy K3s service file
+  template:
+    src: "k3s.service.j2"
+    dest: "{{ systemd_dir }}/k3s-node.service"
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Enable and check K3s service
+  systemd:
+    name: k3s-node
+    daemon_reload: yes
+    state: restarted
+    enabled: yes

ansible/roles/k3s/node/templates/k3s.service.j2

@@ -0,0 +1,24 @@
+[Unit]
+Description=Lightweight Kubernetes
+Documentation=https://k3s.io
+After=network-online.target
+
+[Service]
+Type=notify
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/local/bin/k3s agent --server https://{{ controller_ip }}:6443 --token {{ hostvars[groups['controller'][0]]['token'] }} {{ extra_agent_args | default("") }}
+KillMode=process
+Delegate=yes
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target