From 00eddd548dc342f69e683b5831e6465a26747f2c Mon Sep 17 00:00:00 2001 From: Anish Lakhwara Date: Tue, 5 Sep 2023 14:50:18 -0700 Subject: [PATCH] feat: k3s ansible playbook (#1071) It changes the directory layout of the ansible playbook to a more "best practices" friendly approach using ansible roles and a real inventory file Co-authored-by: Ilya Kreymer --- .../inventory/sample-k3s/group_vars/all.yml | 10 ++ ansible/inventory/sample-k3s/hosts.ini | 12 +++ ansible/playbooks/install_k3s.yml | 30 ++++++ ansible/roles/btrix/install/tasks/main.yml | 55 +++++++++++ ansible/roles/btrix/prereq/tasks/main.yml | 26 ++++++ .../btrix/templates/k8s-manifest.yaml.j2 | 11 +++ ansible/roles/download/tasks/main.yml | 36 ++++++++ ansible/roles/k3s/master/defaults/main.yml | 2 + ansible/roles/k3s/master/tasks/main.yml | 91 +++++++++++++++++++ .../roles/k3s/master/templates/k3s.service.j2 | 24 +++++ ansible/roles/k3s/node/tasks/main.yml | 16 ++++ .../roles/k3s/node/templates/k3s.service.j2 | 24 +++++ ansible/roles/prereq/tasks/main.yml | 54 +++++++++++ ansible/roles/reset/tasks/main.yml | 42 +++++++++ .../reset/tasks/umount_with_children.yml | 16 ++++ docs/deploy/ansible/k3s.md | 51 +++++++++++ docs/deploy/production.md | 1 + 17 files changed, 501 insertions(+) create mode 100644 ansible/inventory/sample-k3s/group_vars/all.yml create mode 100644 ansible/inventory/sample-k3s/hosts.ini create mode 100644 ansible/playbooks/install_k3s.yml create mode 100644 ansible/roles/btrix/install/tasks/main.yml create mode 100644 ansible/roles/btrix/prereq/tasks/main.yml create mode 100644 ansible/roles/btrix/templates/k8s-manifest.yaml.j2 create mode 100644 ansible/roles/download/tasks/main.yml create mode 100644 ansible/roles/k3s/master/defaults/main.yml create mode 100644 ansible/roles/k3s/master/tasks/main.yml create mode 100644 ansible/roles/k3s/master/templates/k3s.service.j2 create mode 100644 ansible/roles/k3s/node/tasks/main.yml create mode 100644 ansible/roles/k3s/node/templates/k3s.service.j2 create mode 100644 ansible/roles/prereq/tasks/main.yml create mode 100644 ansible/roles/reset/tasks/main.yml create mode 100644 ansible/roles/reset/tasks/umount_with_children.yml create mode 100644 docs/deploy/ansible/k3s.md diff --git a/ansible/inventory/sample-k3s/group_vars/all.yml b/ansible/inventory/sample-k3s/group_vars/all.yml new file mode 100644 index 0000000000..65adfc5bb5 --- /dev/null +++ b/ansible/inventory/sample-k3s/group_vars/all.yml @@ -0,0 +1,10 @@ +--- +k3s_version: v1.22.3+k3s1 +ansible_user: debian +systemd_dir: /etc/systemd/system +controller_ip: "{{ hostvars[groups['controller'][0]]['ansible_host'] | default(groups['controller'][0]) }}" +extra_server_args: "--disable traefik" +extra_agent_args: "" +project_name: browsertrix-cloud +domain: my-domain.example.com +email: test@example.com diff --git a/ansible/inventory/sample-k3s/hosts.ini b/ansible/inventory/sample-k3s/hosts.ini new file mode 100644 index 0000000000..e628ac39b6 --- /dev/null +++ b/ansible/inventory/sample-k3s/hosts.ini @@ -0,0 +1,12 @@ +[controller] +# Set to the IP address of the k3s host node +127.0.0.1 + +# Uncomment for multi-node deployment +# [node] +# 192.168.1.2 + +[k3s_cluster:children] +controller +# Uncomment for multi-node deployment +# node diff --git a/ansible/playbooks/install_k3s.yml b/ansible/playbooks/install_k3s.yml new file mode 100644 index 0000000000..4d174d1762 --- /dev/null +++ b/ansible/playbooks/install_k3s.yml @@ -0,0 +1,30 @@ +--- + +# Can be skipped if k3s is installed, this installs k3s +- hosts: k3s_cluster + gather_facts: yes + connection: local # Comment if deploying to remote host + become: yes + roles: + - role: prereq + - role: download + +# Can be skipped if k3s is installed, this configures the master k3s node +- hosts: controller + connection: local # Comment if deploying to remote host + become: yes + roles: + - role: k3s/master + +# Uncomment for multi-node deployment +# - hosts: node +# roles: +# - role: k3s/node + +# Ansible controller to install browsertrix cloud +- hosts: 127.0.0.1 + connection: local + become: yes # Can be removed if not using the btrix/prereq role + roles: + - role: btrix/prereq # Only required if you wish to install & configure Helm / Kubectl + - role: btrix/install diff --git a/ansible/roles/btrix/install/tasks/main.yml b/ansible/roles/btrix/install/tasks/main.yml new file mode 100644 index 0000000000..1c4ef7610e --- /dev/null +++ b/ansible/roles/btrix/install/tasks/main.yml @@ -0,0 +1,55 @@ +--- +- name: Create directory .kube + file: + path: ~{{ ansible_user }}/.kube + state: directory + owner: "{{ ansible_user }}" + mode: "u=rwx,g=rx,o=" + +- name: Check whether kube config exists + stat: + path: ~/.kube/config + register: kubeconfig_result + +- name: Get k3s config + ansible.posix.synchronize: + src: rsync://{{ controller_ip }}/home/{{ ansible_user }}/.kube/config + dest: ~/.kube/config + when: not kubeconfig_result.stat.exists + +- name: Check whether CRDs installed + ansible.builtin.command: kubectl get crd + register: crd_register + +- name: Run all admin bookstrap scripts + ansible.builtin.command: >- + ../chart/admin/logging/scripts/eck_install.sh + register: addons_init + when: "crd_register.stdout | length < 16" + changed_when: true + +- name: Install Cert-Manager + ansible.builtin.command: kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml + changed_when: true + +- name: Install ingress-nginx + ansible.builtin.command: helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace + +- name: Install Metrics Server + ansible.builtin.shell: | + helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/ + helm upgrade --install metrics-server metrics-server/metrics-server + +- name: Output values yaml + ansible.builtin.template: + src: k8s-manifest.yaml.j2 + dest: ../chart/{{ project_name }}-values.yaml + mode: u+rw + +- name: d_ocean | helm | deploy btrix + ansible.builtin.command: helm upgrade --install -f ../chart/values.yaml -f ../chart/{{ project_name }}-values.yaml btrix ../chart/ + register: helm_result + changed_when: helm_result.rc == 0 + environment: + KUBECONFIG: "/home/{{ ansible_user }}/.kube/config" + tags: helm_upgrade diff --git a/ansible/roles/btrix/prereq/tasks/main.yml b/ansible/roles/btrix/prereq/tasks/main.yml new file mode 100644 index 0000000000..eda12595d9 --- /dev/null +++ b/ansible/roles/btrix/prereq/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: Gather installed helm version, if there is any + ansible.builtin.shell: helm version + register: helm_result + failed_when: helm_result.rc != 0 and helm_result.rc != 127 + # Since this is a reporting task, it should never change + # as well run and register a result in any case + changed_when: false + check_mode: false + +- name: Install Helm + ansible.builtin.shell: | + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod +700 get_helm.sh + ./get_helm.sh + when: helm_result.rc != 0 + +- name: Install kubectl + ansible.builtin.shell: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + +- name: Install jq + ansible.builtin.package: + name: jq + state: present diff --git a/ansible/roles/btrix/templates/k8s-manifest.yaml.j2 b/ansible/roles/btrix/templates/k8s-manifest.yaml.j2 new file mode 100644 index 0000000000..50c5080f40 --- /dev/null +++ b/ansible/roles/btrix/templates/k8s-manifest.yaml.j2 @@ -0,0 +1,11 @@ +ingress_class: "nginx" + +mongo_auth: + username: root + password: example + +ingress: + host: "{{ domain }}" + cert_email: "{{ email }}" + scheme: "https" + tls: true diff --git a/ansible/roles/download/tasks/main.yml b/ansible/roles/download/tasks/main.yml new file mode 100644 index 0000000000..1450fd86ee --- /dev/null +++ b/ansible/roles/download/tasks/main.yml @@ -0,0 +1,36 @@ +--- + +- name: Download k3s binary x64 + get_url: + url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s + checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt + dest: /usr/local/bin/k3s + owner: root + group: root + mode: 0755 + when: ansible_facts.architecture == "x86_64" + +- name: Download k3s binary arm64 + get_url: + url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-arm64 + checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm64.txt + dest: /usr/local/bin/k3s + owner: root + group: root + mode: 0755 + when: + - ( ansible_facts.architecture is search("arm") and + ansible_facts.userspace_bits == "64" ) or + ansible_facts.architecture is search("aarch64") + +- name: Download k3s binary armhf + get_url: + url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-armhf + checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm.txt + dest: /usr/local/bin/k3s + owner: root + group: root + mode: 0755 + when: + - ansible_facts.architecture is search("arm") + - ansible_facts.userspace_bits == "32" diff --git a/ansible/roles/k3s/master/defaults/main.yml b/ansible/roles/k3s/master/defaults/main.yml new file mode 100644 index 0000000000..c56778f930 --- /dev/null +++ b/ansible/roles/k3s/master/defaults/main.yml @@ -0,0 +1,2 @@ +--- +k3s_server_location: /var/lib/rancher/k3s diff --git a/ansible/roles/k3s/master/tasks/main.yml b/ansible/roles/k3s/master/tasks/main.yml new file mode 100644 index 0000000000..b56ea81766 --- /dev/null +++ b/ansible/roles/k3s/master/tasks/main.yml @@ -0,0 +1,91 @@ +--- + +- name: Copy K3s service file + register: k3s_service + template: + src: "k3s.service.j2" + dest: "{{ systemd_dir }}/k3s.service" + owner: root + group: root + mode: 0644 + +- name: Enable and check K3s service + systemd: + name: k3s + daemon_reload: yes + state: started + enabled: yes + +- name: Wait for node-token + wait_for: + path: "{{ k3s_server_location }}/server/node-token" + +- name: Register node-token file access mode + stat: + path: "{{ k3s_server_location }}/server/node-token" + register: p + +- name: Change file access node-token + file: + path: "{{ k3s_server_location }}/server/node-token" + mode: "g+rx,o+rx" + +- name: Read node-token from master + slurp: + path: "{{ k3s_server_location }}/server/node-token" + register: node_token + +- name: Store Master node-token + set_fact: + token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}" + +- name: Restore node-token file access + file: + path: "{{ k3s_server_location }}/server/node-token" + mode: "{{ p.stat.mode }}" + +- name: Create directory .kube + file: + path: ~{{ ansible_user }}/.kube + state: directory + owner: "{{ ansible_user }}" + mode: "u=rwx,g=rx,o=" + +- name: Copy config file to user home directory + copy: + src: /etc/rancher/k3s/k3s.yaml + dest: ~{{ ansible_user }}/.kube/config + remote_src: yes + owner: "{{ ansible_user }}" + mode: "u=rw,g=,o=" + +- name: Replace https://localhost:6443 by https://controller-ip:6443 + command: >- + k3s kubectl config set-cluster default + --server=https://{{ controller_ip }}:6443 + --kubeconfig ~{{ ansible_user }}/.kube/config + changed_when: true + +- name: Check that the kubectl binary exists + stat: + path: /usr/local/bin/kubectl + register: kubectl_result + +- name: Check that the crictl binary exists + stat: + path: /usr/local/bin/crictl + register: crictl_result + +- name: Create kubectl symlink + file: + src: /usr/local/bin/k3s + dest: /usr/local/bin/kubectl + state: link + when: not kubectl_result.stat.exists + +- name: Create crictl symlink + file: + src: /usr/local/bin/k3s + dest: /usr/local/bin/crictl + state: link + when: not crictl_result.stat.exists diff --git a/ansible/roles/k3s/master/templates/k3s.service.j2 b/ansible/roles/k3s/master/templates/k3s.service.j2 new file mode 100644 index 0000000000..a56ab10840 --- /dev/null +++ b/ansible/roles/k3s/master/templates/k3s.service.j2 @@ -0,0 +1,24 @@ +[Unit] +Description=Lightweight Kubernetes +Documentation=https://k3s.io +After=network-online.target + +[Service] +Type=notify +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args | default("") }} +KillMode=process +Delegate=yes +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/k3s/node/tasks/main.yml b/ansible/roles/k3s/node/tasks/main.yml new file mode 100644 index 0000000000..0ce8e08d0f --- /dev/null +++ b/ansible/roles/k3s/node/tasks/main.yml @@ -0,0 +1,16 @@ +--- + +- name: Copy K3s service file + template: + src: "k3s.service.j2" + dest: "{{ systemd_dir }}/k3s-node.service" + owner: root + group: root + mode: 0755 + +- name: Enable and check K3s service + systemd: + name: k3s-node + daemon_reload: yes + state: restarted + enabled: yes diff --git a/ansible/roles/k3s/node/templates/k3s.service.j2 b/ansible/roles/k3s/node/templates/k3s.service.j2 new file mode 100644 index 0000000000..070f945f85 --- /dev/null +++ b/ansible/roles/k3s/node/templates/k3s.service.j2 @@ -0,0 +1,24 @@ +[Unit] +Description=Lightweight Kubernetes +Documentation=https://k3s.io +After=network-online.target + +[Service] +Type=notify +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s agent --server https://{{ controller_ip }}:6443 --token {{ hostvars[groups['controller'][0]]['token'] }} {{ extra_agent_args | default("") }} +KillMode=process +Delegate=yes +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/prereq/tasks/main.yml b/ansible/roles/prereq/tasks/main.yml new file mode 100644 index 0000000000..bed7d3edc4 --- /dev/null +++ b/ansible/roles/prereq/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Set SELinux to disabled state + selinux: + state: disabled + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux', 'Rocky Linux'] + +- name: Enable IPv4 forwarding + sysctl: + name: net.ipv4.ip_forward + value: "1" + state: present + reload: yes + +- name: Enable IPv6 forwarding + sysctl: + name: net.ipv6.conf.all.forwarding + value: "1" + state: present + reload: yes + when: ansible_all_ipv6_addresses + +- name: Add br_netfilter to /etc/modules-load.d/ + copy: + content: "br_netfilter" + dest: /etc/modules-load.d/br_netfilter.conf + mode: "u=rw,g=,o=" + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux', 'Rocky Linux'] + +- name: Load br_netfilter + modprobe: + name: br_netfilter + state: present + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux', 'Rocky Linux'] + +- name: Set bridge-nf-call-iptables (just to be sure) + sysctl: + name: "{{ item }}" + value: "1" + state: present + reload: yes + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux', 'Rocky Linux'] + loop: + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-ip6tables + +- name: Add /usr/local/bin to sudo secure_path + lineinfile: + line: 'Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin' + regexp: "Defaults(\\s)*secure_path(\\s)*=" + state: present + insertafter: EOF + path: /etc/sudoers + validate: 'visudo -cf %s' + when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux', 'Rocky Linux'] diff --git a/ansible/roles/reset/tasks/main.yml b/ansible/roles/reset/tasks/main.yml new file mode 100644 index 0000000000..728447fbb3 --- /dev/null +++ b/ansible/roles/reset/tasks/main.yml @@ -0,0 +1,42 @@ +--- +- name: Disable services + systemd: + name: "{{ item }}" + state: stopped + enabled: no + failed_when: false + with_items: + - k3s + - k3s-node + +- name: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" + register: pkill_containerd_shim_runc + command: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" + changed_when: "pkill_containerd_shim_runc.rc == 0" + failed_when: false + +- name: Umount k3s filesystems + include_tasks: umount_with_children.yml + with_items: + - /run/k3s + - /var/lib/kubelet + - /run/netns + - /var/lib/rancher/k3s + loop_control: + loop_var: mounted_fs + +- name: Remove service files, binaries and data + file: + name: "{{ item }}" + state: absent + with_items: + - /usr/local/bin/k3s + - "{{ systemd_dir }}/k3s.service" + - "{{ systemd_dir }}/k3s-node.service" + - /etc/rancher/k3s + - /var/lib/kubelet + - /var/lib/rancher/k3s + +- name: daemon_reload + systemd: + daemon_reload: yes diff --git a/ansible/roles/reset/tasks/umount_with_children.yml b/ansible/roles/reset/tasks/umount_with_children.yml new file mode 100644 index 0000000000..5883b70a64 --- /dev/null +++ b/ansible/roles/reset/tasks/umount_with_children.yml @@ -0,0 +1,16 @@ +--- +- name: Get the list of mounted filesystems + shell: set -o pipefail && cat /proc/mounts | awk '{ print $2}' | grep -E "^{{ mounted_fs }}" + register: get_mounted_filesystems + args: + executable: /bin/bash + failed_when: false + changed_when: get_mounted_filesystems.stdout | length > 0 + check_mode: false + +- name: Umount filesystem + mount: + path: "{{ item }}" + state: unmounted + with_items: + "{{ get_mounted_filesystems.stdout_lines | reverse | list }}" diff --git a/docs/deploy/ansible/k3s.md b/docs/deploy/ansible/k3s.md new file mode 100644 index 0000000000..badc103b86 --- /dev/null +++ b/docs/deploy/ansible/k3s.md @@ -0,0 +1,51 @@ +# K3S + +*Playbook Path: [ansible/playbooks/install_k3s.yml](https://github.com/webrecorder/browsertrix-cloud/blob/main/ansible/playbooks/install_k3s.yml)* + +This playbook provides an easy way to install Browsertrix Cloud on a Linux box (tested on Rocky Linux 9). It automatically sets up Browsertrix with Let's Encrypt certificates. + +### Requirements + +To run this ansible playbook, you need to: + +* Have a server / VPS where browsertrix will run. +* Configure a DNS A Record to point at your server's IP address. +* Make sure you can ssh to it, with a sudo user: ssh @ +* Install Ansible on your local machine (the control machine). + + +1. Clone the repo: +```zsh +git clone https://github.com/webrecorder/browsertrix-cloud.git +cd browsertrix-cloud +``` + +2. Optional: Create a copy of the [inventory directory] and name it what you like (alternatively edit the sample files in place) +```zsh +cp -r ansible/inventory/sample-k3s ansible/inventory/my-deployment +``` + +1. [Look at the configuration options](https://github.com/webrecorder/browsertrix-cloud/blob/main/ansible/inventory/sample-k3s/group_vars/all.yml) and modify them to match your setup + +2. Change the [hosts IP address](https://github.com/webrecorder/browsertrix-cloud/blob/main/ansible/inventory/sample-k3s/hosts.ini) in your just created inventory + +3. Copy the playbook into the root ansible directory: +```zsh +cp ansible/playbooks/install_k3s.yml ansible/install_k3s.yml +``` + +5. You may need to make modifications to the playbook itself based on your configuration. The playbook lists sections that can be removed or changed based on whether you'd like to install a multi-node or single-node k3s installation for your Browsertrix Cloud deployment. By default the playbook assumes you'll run in a single-node environment deploying directly to `localhost` + +6. Run the playbook: +```zsh +ansible-playbook -i inventory/my-deployment/hosts.ini install_k3s.yml +``` + +#### Upgrading + +1. Run `git pull` + +2. Run the playbook: +```zsh +ansible-playbook -i inventory/hosts playbooks/install_k3s.yml -t helm_upgrade +``` diff --git a/docs/deploy/production.md b/docs/deploy/production.md index 1e7f416acd..2f0ec869c3 100644 --- a/docs/deploy/production.md +++ b/docs/deploy/production.md @@ -75,3 +75,4 @@ Currently, we provide playbooks for the following tested environments: - [DigitalOcean](ansible/digitalocean.md) - [Microk8s](ansible/microk8s.md) +- [k3s](ansible/k3s.md)