From d05241742e353c8b952d159271d6883583d78c8d Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 15 Oct 2023 13:33:22 +0200 Subject: [PATCH 01/48] Explicitly install iptables --- bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap.sh b/bootstrap.sh index 2127cb9b..e272255c 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -65,6 +65,7 @@ install_dependencies_debian() { python3-pip aptitude direnv + iptables ) REQUIRED_PACKAGES_ARM64=( From 0e0624558e09f6a8334b8acd970fcca34e9e070d Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 15 Oct 2023 14:03:48 +0200 Subject: [PATCH 02/48] Adjust ansible-lint, attempt to fix syntax error when password contains quotes --- .ansible-lint | 1 + bootstrap.sh | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.ansible-lint b/.ansible-lint index 29a410b5..824d2c33 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -6,6 +6,7 @@ warn_list: - package-latest - yaml - fqcn + - var-naming[no-role-prefix] skip_list: - command-instead-of-shell - experimental diff --git a/bootstrap.sh b/bootstrap.sh index e272255c..bf10b006 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -393,7 +393,12 @@ else echo "email_password: \"${email_password}\"" >> $HOME/ansible-easy-vpn/secret.yml fi -echo "user_password: \"${user_password}\"" >> $HOME/ansible-easy-vpn/secret.yml +if [[ $user_password =~ '"' ]]; then + echo "user_password: '${user_password}'" >> $HOME/ansible-easy-vpn/secret.yml +else + echo "user_password: \"${user_password}\"" >> $HOME/ansible-easy-vpn/secret.yml +fi + jwt_secret=$(openssl rand -hex 23) session_secret=$(openssl rand -hex 23) From 48df9cd13ed1caffae8d67b2b04f034feccd8553 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 10 Dec 2023 10:47:44 +0100 Subject: [PATCH 03/48] Improve iptables compatibility, add Debian 12 testing --- .github/workflows/ci.yml | 16 ++++++++++++++++ .github/workflows/matrix_includes.json | 5 +++++ roles/system/tasks/firewall.yml | 13 ++++++++++--- .../systemd/iptables.service.j2} | 2 +- 4 files changed, 32 insertions(+), 4 deletions(-) rename roles/system/{files/iptables.service => templates/systemd/iptables.service.j2} (69%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4c95d26..54b8f078 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,12 @@ on: type: boolean default: false + only_debian_12: + description: "Only run on Debian 12" + required: false + type: boolean + default: false + manual_mode: description: "Don't destroy the server after the setup is complete" required: false @@ -70,6 +76,9 @@ jobs: elif [[ ${ONLY_DEBIAN_11} == 'true' ]]; then # Only deploy on Debian 11, don't use Letsencrypt Staging matrix=$(jq 'map(. | select((.os=="debian-11")) )' .github/workflows/matrix_includes.json) + elif [[ ${ONLY_DEBIAN_12} == 'true' ]]; then + # Only deploy on Debian 11, don't use Letsencrypt Staging + matrix=$(jq 'map(. | select((.os=="debian-12")) )' .github/workflows/matrix_includes.json) else # Deploy on all supported OSes, use Letsencrypt Staging to avoid rate-limiting matrix=$(jq 'map(.)' .github/workflows/matrix_includes.json) @@ -80,6 +89,7 @@ jobs: ONLY_UBUNTU_22: ${{ inputs.only_ubuntu_22 }} ONLY_UBUNTU_20: ${{ inputs.only_ubuntu_20 }} ONLY_DEBIAN_11: ${{ inputs.only_debian_11 }} + ONLY_DEBIAN_11: ${{ inputs.only_debian_12 }} build: runs-on: ubuntu-latest @@ -121,6 +131,10 @@ jobs: echo "EASYVPN_USERNAME_3=$EASYVPN_USERNAME" >> $GITHUB_OUTPUT echo "EASYVPN_PASSWORD_3=$EASYVPN_PASSWORD" >> $GITHUB_OUTPUT ;; + "4") + echo "EASYVPN_USERNAME_4=$EASYVPN_USERNAME" >> $GITHUB_OUTPUT + echo "EASYVPN_PASSWORD_4=$EASYVPN_PASSWORD" >> $GITHUB_OUTPUT + ;; *) exit 1 ;; @@ -272,9 +286,11 @@ jobs: EASYVPN_USERNAME_1: "${{ steps.random_username.outputs.EASYVPN_USERNAME_1 }}" EASYVPN_USERNAME_2: "${{ steps.random_username.outputs.EASYVPN_USERNAME_2 }}" EASYVPN_USERNAME_3: "${{ steps.random_username.outputs.EASYVPN_USERNAME_3 }}" + EASYVPN_USERNAME_4: "${{ steps.random_username.outputs.EASYVPN_USERNAME_4 }}" EASYVPN_PASSWORD_1: "${{ steps.random_username.outputs.EASYVPN_PASSWORD_1 }}" EASYVPN_PASSWORD_2: "${{ steps.random_username.outputs.EASYVPN_PASSWORD_2 }}" EASYVPN_PASSWORD_3: "${{ steps.random_username.outputs.EASYVPN_PASSWORD_3 }}" + EASYVPN_PASSWORD_4: "${{ steps.random_username.outputs.EASYVPN_PASSWORD_4 }}" fetch_config: runs-on: ubuntu-latest diff --git a/.github/workflows/matrix_includes.json b/.github/workflows/matrix_includes.json index fc997b1b..bff1f738 100644 --- a/.github/workflows/matrix_includes.json +++ b/.github/workflows/matrix_includes.json @@ -10,5 +10,10 @@ { "os":"debian-11", "index":3 + }, + { + "os":"debian-12", + "index":4 } + ] diff --git a/roles/system/tasks/firewall.yml b/roles/system/tasks/firewall.yml index f28cac44..f9a5f5f7 100644 --- a/roles/system/tasks/firewall.yml +++ b/roles/system/tasks/firewall.yml @@ -7,9 +7,16 @@ group: root mode: "0644" -- name: Copy the iptables systemd service - ansible.builtin.copy: - src: files/iptables.service +- name: Check the location of iptables-restore + register: "iptables_restore" + changed_when: no + failed_when: "iptables_restore is ''" + ansible.builtin.command: + cmd: "which iptables-restore" + +- name: Template the iptables systemd service + ansible.builtin.template: + src: systemd/iptables.service.j2 dest: /etc/systemd/system/iptables.service owner: root group: root diff --git a/roles/system/files/iptables.service b/roles/system/templates/systemd/iptables.service.j2 similarity index 69% rename from roles/system/files/iptables.service rename to roles/system/templates/systemd/iptables.service.j2 index 2e9b9183..5eb81608 100644 --- a/roles/system/files/iptables.service +++ b/roles/system/templates/systemd/iptables.service.j2 @@ -4,7 +4,7 @@ Before=network-pre.target [Service] Type=oneshot -ExecStart=/sbin/iptables-restore -n /etc/iptables.conf +ExecStart={{ iptables_restore.stdout }} -n /etc/iptables.conf [Install] WantedBy=multi-user.target From 9df334b02ca81fe5423f5ed3c40a33bcd613d87e Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 10 Dec 2023 10:58:22 +0100 Subject: [PATCH 04/48] Add watchtower to automatically update Docker containers --- roles/authelia/defaults/main.yml | 2 +- roles/bunkerweb/defaults/main.yml | 2 +- roles/watchtower/tasks/main.yml | 13 +++++++++++++ roles/wireguard/defaults/main.yml | 2 +- run.yml | 4 ++++ 5 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 roles/watchtower/tasks/main.yml diff --git a/roles/authelia/defaults/main.yml b/roles/authelia/defaults/main.yml index d08fcb95..a297232d 100644 --- a/roles/authelia/defaults/main.yml +++ b/roles/authelia/defaults/main.yml @@ -1,2 +1,2 @@ --- -authelia_version: "4.36" +authelia_version: "latest" diff --git a/roles/bunkerweb/defaults/main.yml b/roles/bunkerweb/defaults/main.yml index 058c327b..a1a41e02 100644 --- a/roles/bunkerweb/defaults/main.yml +++ b/roles/bunkerweb/defaults/main.yml @@ -1,2 +1,2 @@ --- -bunkerweb_version: "1.4.2" +bunkerweb_version: "latest" diff --git a/roles/watchtower/tasks/main.yml b/roles/watchtower/tasks/main.yml new file mode 100644 index 00000000..4fdcfa9c --- /dev/null +++ b/roles/watchtower/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Make sure Watchtower is created and running + register: watchtower_result + retries: 5 + until: authelia_result is succeeded + community.general.docker_container: + name: "watchtower" + image: "containrrr/watchtower" + pull: yes + state: "started" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + restart_policy: unless-stopped diff --git a/roles/wireguard/defaults/main.yml b/roles/wireguard/defaults/main.yml index 74565252..cfb664c2 100644 --- a/roles/wireguard/defaults/main.yml +++ b/roles/wireguard/defaults/main.yml @@ -1,2 +1,2 @@ --- -wg_easy_version: "7" +wg_easy_version: "latest" diff --git a/run.yml b/run.yml index 252afdec..4e4d50cf 100755 --- a/run.yml +++ b/run.yml @@ -41,6 +41,10 @@ tags: - dns + - role: watchtower + tags: + - watchtower + - role: authelia tags: - authelia From 9bb4140d86297061ac0e3acd0bdb83ae1bdcad13 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 10 Dec 2023 10:59:20 +0100 Subject: [PATCH 05/48] Disable Adguard logs and anonymize client IPs by default --- roles/dns/files/AdGuardHome.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/dns/files/AdGuardHome.yaml b/roles/dns/files/AdGuardHome.yaml index 305346df..7f773968 100644 --- a/roles/dns/files/AdGuardHome.yaml +++ b/roles/dns/files/AdGuardHome.yaml @@ -14,11 +14,11 @@ dns: - 0.0.0.0 port: 53 statistics_interval: 1 - querylog_enabled: true - querylog_file_enabled: true + querylog_enabled: false + querylog_file_enabled: false querylog_interval: 2160h querylog_size_memory: 1000 - anonymize_client_ip: false + anonymize_client_ip: true protection_enabled: true blocking_mode: default blocking_ipv4: "" From ab8428e6529e95ea56b9e928abb483731ed1a800 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 10 Dec 2023 11:03:12 +0100 Subject: [PATCH 06/48] Redirect to the Wireguard host by default after succesfull auth --- roles/authelia/templates/configuration.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/authelia/templates/configuration.yml.j2 b/roles/authelia/templates/configuration.yml.j2 index 33451001..9cf761cf 100755 --- a/roles/authelia/templates/configuration.yml.j2 +++ b/roles/authelia/templates/configuration.yml.j2 @@ -4,7 +4,7 @@ ############################################################### jwt_secret: {{ jwt_secret }} -default_redirection_url: https://{{ authelia_host }} +default_redirection_url: https://{{ wireguard_host }} ntp: disable_failure: true From 9aef468f6c1d3cb8a68e4a0f0a13e0bf12670cc4 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 10 Dec 2023 11:04:05 +0100 Subject: [PATCH 07/48] Fixup --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54b8f078..67c89fdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: ONLY_UBUNTU_22: ${{ inputs.only_ubuntu_22 }} ONLY_UBUNTU_20: ${{ inputs.only_ubuntu_20 }} ONLY_DEBIAN_11: ${{ inputs.only_debian_11 }} - ONLY_DEBIAN_11: ${{ inputs.only_debian_12 }} + ONLY_DEBIAN_12: ${{ inputs.only_debian_12 }} build: runs-on: ubuntu-latest From cc1997a1db6feb4ddea6e435922b558dff4e7a15 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 10 Dec 2023 11:12:41 +0100 Subject: [PATCH 08/48] Fixup --- roles/system/tasks/firewall.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/system/tasks/firewall.yml b/roles/system/tasks/firewall.yml index f9a5f5f7..f9bbf197 100644 --- a/roles/system/tasks/firewall.yml +++ b/roles/system/tasks/firewall.yml @@ -10,7 +10,7 @@ - name: Check the location of iptables-restore register: "iptables_restore" changed_when: no - failed_when: "iptables_restore is ''" + failed_when: "iptables_restore.stdout | length == 0" ansible.builtin.command: cmd: "which iptables-restore" From 9935b1b9481d2ee055193616707a4710e52f1407 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 10 Dec 2023 11:19:49 +0100 Subject: [PATCH 09/48] Fixup --- roles/watchtower/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/watchtower/tasks/main.yml b/roles/watchtower/tasks/main.yml index 4fdcfa9c..6b0b9cdc 100644 --- a/roles/watchtower/tasks/main.yml +++ b/roles/watchtower/tasks/main.yml @@ -2,7 +2,7 @@ - name: Make sure Watchtower is created and running register: watchtower_result retries: 5 - until: authelia_result is succeeded + until: watchtower_result is succeeded community.general.docker_container: name: "watchtower" image: "containrrr/watchtower" From 23503e661722dab5dff0c4cf6e30cab96276d3a9 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 10 Dec 2023 11:31:41 +0100 Subject: [PATCH 10/48] Add missing matrix for debian 12 --- .github/workflows/ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67c89fdd..8c6e7761 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -276,6 +276,15 @@ jobs: name: "private-ssh-key-3" path: "id_vpn" + - name: Archive the private SSH key (Matrix 4) + if: ${{ matrix.index == '4' }} + uses: actions/upload-artifact@v3 + with: + name: "private-ssh-key-4" + path: "id_vpn" + + + @@ -339,6 +348,15 @@ jobs: name: "private-ssh-key-3" path: /home/runner/.ssh + - name: Get the private SSH key artifact (Matrix 4) + if: matrix.index == '4' + uses: actions/download-artifact@v3 + with: + name: "private-ssh-key-4" + path: /home/runner/.ssh + + + - name: Set the correct permissions for the SSH key run: | chmod 700 $HOME/.ssh From 0a43ac39b8fe4397ba7095c4bc7a6e72a039b2a0 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 10 Dec 2023 12:46:18 +0100 Subject: [PATCH 11/48] Roll back the redirection URI --- roles/authelia/templates/configuration.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/authelia/templates/configuration.yml.j2 b/roles/authelia/templates/configuration.yml.j2 index 9cf761cf..33451001 100755 --- a/roles/authelia/templates/configuration.yml.j2 +++ b/roles/authelia/templates/configuration.yml.j2 @@ -4,7 +4,7 @@ ############################################################### jwt_secret: {{ jwt_secret }} -default_redirection_url: https://{{ wireguard_host }} +default_redirection_url: https://{{ authelia_host }} ntp: disable_failure: true From a47c05eb209f70ed362ddaed2f755bce4f2be707 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 31 Dec 2023 19:20:26 +0100 Subject: [PATCH 12/48] Switch to GH releases for the wg-easy docker image --- roles/wireguard/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/wireguard/tasks/main.yml b/roles/wireguard/tasks/main.yml index 076636c0..4b00c531 100755 --- a/roles/wireguard/tasks/main.yml +++ b/roles/wireguard/tasks/main.yml @@ -27,7 +27,7 @@ until: wireguard_result is succeeded community.general.docker_container: name: "wg-easy" - image: "weejewel/wg-easy:{{ wg_easy_version }}" + image: "ghcr.io/wg-easy/wg-easy:{{ wg_easy_version }}" pull: yes networks: - name: wg_network From e133a25c4823c0e6db7c44554e9f4b6e06a2f8a5 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 31 Dec 2023 20:14:39 +0100 Subject: [PATCH 13/48] Upload selenium screenshots as artifacts --- .github/workflows/ci.yml | 7 +++++++ roles/bunkerweb/templates/env.j2 | 2 ++ testing/selenium/acceptance.py | 11 +++++++++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c6e7761..88fe2a78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -397,6 +397,13 @@ jobs: EASYVPN_PASSWORD: "${{ needs.build.outputs[format('EASYVPN_PASSWORD_{0}', matrix.index)] }}" DOMAIN: "${{ needs.build.outputs[format('EASYVPN_USERNAME_{0}', matrix.index)] }}.${{ secrets.CLOUDFLARE_DOMAIN }}" + - name: Upload Selenium testing screenshots + if: always() + uses: actions/upload-artifact@v4 + with: + name: "Screenshots" + path: "screenshots/" + destroy: runs-on: ubuntu-latest environment: cicd diff --git a/roles/bunkerweb/templates/env.j2 b/roles/bunkerweb/templates/env.j2 index 56ad05f1..72cef4d2 100644 --- a/roles/bunkerweb/templates/env.j2 +++ b/roles/bunkerweb/templates/env.j2 @@ -33,6 +33,7 @@ REVERSE_PROXY_HEADERS_999=X-Original-URL $scheme://$http_host$request_uri;Conten {{ wireguard_host }}_REVERSE_PROXY_AUTH_REQUEST=/authelia {{ wireguard_host }}_REVERSE_PROXY_AUTH_REQUEST_SIGNIN_URL=https://{{ authelia_host }}/?rd=$scheme%3A%2F%2F$host$request_uri {{ wireguard_host }}_REVERSE_PROXY_AUTH_REQUEST_SET=$user $upstream_http_remote_user;$groups $upstream_http_remote_groups;$name $upstream_http_remote_name;$email $upstream_http_remote_email +{{ wireguard_host }}_REVERSE_PROXY_HEADERS=Remote-User $$user;Remote-Groups $$groups;Remote-Name $$name;Remote-Email $$email # Adguard {% if enable_adguard_unbound_doh %} {{ adguard_host }}_REVERSE_PROXY_URL=/ @@ -40,4 +41,5 @@ REVERSE_PROXY_HEADERS_999=X-Original-URL $scheme://$http_host$request_uri;Conten {{ adguard_host }}_REVERSE_PROXY_AUTH_REQUEST=/authelia {{ adguard_host }}_REVERSE_PROXY_AUTH_REQUEST_SIGNIN_URL=https://{{ authelia_host }}/?rd=$scheme%3A%2F%2F$host$request_uri {{ adguard_host }}_REVERSE_PROXY_AUTH_REQUEST_SET=$user $upstream_http_remote_user;$groups $upstream_http_remote_groups;$name $upstream_http_remote_name;$email $upstream_http_remote_email +{{ adguard_host }}_REVERSE_PROXY_HEADERS=Remote-User $$user;Remote-Groups $$groups;Remote-Name $$name;Remote-Email $$email {% endif %} diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 371dda5a..cb08ca5c 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -5,8 +5,10 @@ from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.action_chains import ActionChains +from selenium.webdriver.common.exceptions import WebDriverException from time import sleep +from os import mkdir import argparse import logging import pyotp @@ -46,7 +48,7 @@ def register_2fa(driver, base_url, username, password, ssh_agent): - logger.debug(f"Fetching {base_url}") + logger.debug(f"Fetching wg.{base_url}") driver.get(f"https://wg.{base_url}") sleep(0.5) logger.debug(f"Filling out the username field with {username}") @@ -106,7 +108,12 @@ def register_2fa(driver, base_url, username, password, ssh_agent): def download_wg_config(driver, base_url, client): logger.debug(f"Opening wg.{base_url} in the browser") - driver.get(f"https://wg.{base_url}") + try: + driver.get(f"https://wg.{base_url}") + except WebDriverException: + mkdir("screenshots") + driver.save_screenshot("screenshots/ss.png") + exit(0) sleep(2) logger.debug("Clicking on the 'New Client' button") new_client_button = driver.find_element("xpath", "//*[contains(text(), 'New Client')]") From d56ef71b93aa395ac9143183cbe7d656d6ee1761 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 31 Dec 2023 20:27:44 +0100 Subject: [PATCH 14/48] Fixup --- testing/selenium/acceptance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index cb08ca5c..f3507541 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -5,7 +5,7 @@ from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.action_chains import ActionChains -from selenium.webdriver.common.exceptions import WebDriverException +from selenium.common.exceptions import WebDriverException from time import sleep from os import mkdir From 668ad99e86874e90663ff99e83c0f19105b51c6a Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 31 Dec 2023 20:39:56 +0100 Subject: [PATCH 15/48] Fixup --- testing/selenium/acceptance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index f3507541..6e9899bb 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -110,7 +110,7 @@ def download_wg_config(driver, base_url, client): logger.debug(f"Opening wg.{base_url} in the browser") try: driver.get(f"https://wg.{base_url}") - except WebDriverException: + except: mkdir("screenshots") driver.save_screenshot("screenshots/ss.png") exit(0) From c1bb290bfc8faffaf6aa1246c9ccc1d43c1b8f3a Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 31 Dec 2023 20:53:44 +0100 Subject: [PATCH 16/48] Fixup --- testing/selenium/acceptance.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 6e9899bb..fa3ede8d 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -49,7 +49,12 @@ def register_2fa(driver, base_url, username, password, ssh_agent): logger.debug(f"Fetching wg.{base_url}") - driver.get(f"https://wg.{base_url}") + try: + driver.get(f"https://wg.{base_url}") + except: + mkdir("screenshots") + driver.save_screenshot("screenshots/ss.png") + exit(1) sleep(0.5) logger.debug(f"Filling out the username field with {username}") username_field = driver.find_element("id", "username-textfield") @@ -108,12 +113,7 @@ def register_2fa(driver, base_url, username, password, ssh_agent): def download_wg_config(driver, base_url, client): logger.debug(f"Opening wg.{base_url} in the browser") - try: - driver.get(f"https://wg.{base_url}") - except: - mkdir("screenshots") - driver.save_screenshot("screenshots/ss.png") - exit(0) + driver.get(f"https://wg.{base_url}") sleep(2) logger.debug("Clicking on the 'New Client' button") new_client_button = driver.find_element("xpath", "//*[contains(text(), 'New Client')]") From 89bc72c02a8669d3e88d518a9d7a5ca3ed1e9476 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 31 Dec 2023 21:38:56 +0100 Subject: [PATCH 17/48] Revert bunkerweb to 1.4.2 --- roles/bunkerweb/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bunkerweb/defaults/main.yml b/roles/bunkerweb/defaults/main.yml index a1a41e02..058c327b 100644 --- a/roles/bunkerweb/defaults/main.yml +++ b/roles/bunkerweb/defaults/main.yml @@ -1,2 +1,2 @@ --- -bunkerweb_version: "latest" +bunkerweb_version: "1.4.2" From dce9db5a6a4d83b05a7f3f76f4e0fbfc9f3a7195 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Sun, 31 Dec 2023 21:54:16 +0100 Subject: [PATCH 18/48] Fixup --- roles/bunkerweb/templates/env.j2 | 2 -- testing/selenium/acceptance.py | 8 +------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/roles/bunkerweb/templates/env.j2 b/roles/bunkerweb/templates/env.j2 index 72cef4d2..56ad05f1 100644 --- a/roles/bunkerweb/templates/env.j2 +++ b/roles/bunkerweb/templates/env.j2 @@ -33,7 +33,6 @@ REVERSE_PROXY_HEADERS_999=X-Original-URL $scheme://$http_host$request_uri;Conten {{ wireguard_host }}_REVERSE_PROXY_AUTH_REQUEST=/authelia {{ wireguard_host }}_REVERSE_PROXY_AUTH_REQUEST_SIGNIN_URL=https://{{ authelia_host }}/?rd=$scheme%3A%2F%2F$host$request_uri {{ wireguard_host }}_REVERSE_PROXY_AUTH_REQUEST_SET=$user $upstream_http_remote_user;$groups $upstream_http_remote_groups;$name $upstream_http_remote_name;$email $upstream_http_remote_email -{{ wireguard_host }}_REVERSE_PROXY_HEADERS=Remote-User $$user;Remote-Groups $$groups;Remote-Name $$name;Remote-Email $$email # Adguard {% if enable_adguard_unbound_doh %} {{ adguard_host }}_REVERSE_PROXY_URL=/ @@ -41,5 +40,4 @@ REVERSE_PROXY_HEADERS_999=X-Original-URL $scheme://$http_host$request_uri;Conten {{ adguard_host }}_REVERSE_PROXY_AUTH_REQUEST=/authelia {{ adguard_host }}_REVERSE_PROXY_AUTH_REQUEST_SIGNIN_URL=https://{{ authelia_host }}/?rd=$scheme%3A%2F%2F$host$request_uri {{ adguard_host }}_REVERSE_PROXY_AUTH_REQUEST_SET=$user $upstream_http_remote_user;$groups $upstream_http_remote_groups;$name $upstream_http_remote_name;$email $upstream_http_remote_email -{{ adguard_host }}_REVERSE_PROXY_HEADERS=Remote-User $$user;Remote-Groups $$groups;Remote-Name $$name;Remote-Email $$email {% endif %} diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index fa3ede8d..598fd20a 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -5,7 +5,6 @@ from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.action_chains import ActionChains -from selenium.common.exceptions import WebDriverException from time import sleep from os import mkdir @@ -49,12 +48,7 @@ def register_2fa(driver, base_url, username, password, ssh_agent): logger.debug(f"Fetching wg.{base_url}") - try: - driver.get(f"https://wg.{base_url}") - except: - mkdir("screenshots") - driver.save_screenshot("screenshots/ss.png") - exit(1) + driver.get(f"https://wg.{base_url}") sleep(0.5) logger.debug(f"Filling out the username field with {username}") username_field = driver.find_element("id", "username-textfield") From 7239a9e41b41a3a952fcf49ada3cf288f87b23fc Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Tue, 2 Apr 2024 10:49:19 +0200 Subject: [PATCH 19/48] Modify testing workflow, bump unbound, alpine and adguard versions, fix f2b on Debian 12 --- .github/workflows/ci.yml | 18 +++++++----------- roles/dns/templates/Dockerfile | 6 +++--- roles/fail2ban/templates/jail.local.j2 | 1 + 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88fe2a78..b7d933ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,8 +161,6 @@ jobs: - name: Install git and expect (Debian-based) run: ssh root@$SERVER_IPV4 apt install -y git expect wamerican - - - uses: infraway/create-dns-record@v2.0 with: type: "A" @@ -429,15 +427,13 @@ jobs: - name: Delete all Cloudflare domains run: >- - curl -s -X GET https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?per_page=500 - -H "Authorization: Bearer $CLOUDFLARE_TOKEN" - -H "Content-Type: application/json" | - jq .result[].id | - tr -d '"' | - ( while read id; do curl -s -X DELETE - https://api.cloudflare.com/client/v4/zones/5420f91fefac252d89d9495a8d35ae73/dns_records/${id} - -H "Authorization: Bearer $CLOUDFLARE_TOKEN" - -H "Content-Type: application/json"; done; ) + curl --silent "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records?per_page=50000" \ + --header "Authorization: Bearer $CLOUDFLARE_TOKEN" \ + | jq --raw-output '.result[].id' | while read id + do + curl --silent --request DELETE "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records/$id" \ + --header "Authorization: Bearer $CLOUDFLARE_TOKEN" + done env: CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }} ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE }} diff --git a/roles/dns/templates/Dockerfile b/roles/dns/templates/Dockerfile index 4595f365..0557dfea 100644 --- a/roles/dns/templates/Dockerfile +++ b/roles/dns/templates/Dockerfile @@ -1,9 +1,9 @@ -FROM alpine:3.15 +FROM alpine:3.18 RUN apk add --no-cache \ libcap \ - unbound=1.13.2-r2 \ + unbound=1.19.3-r0 \ dnscrypt-proxy WORKDIR /tmp @@ -13,7 +13,7 @@ RUN wget https://www.internic.net/domain/named.root -qO- >> /etc/unbound/root.hi COPY files/ /opt/ # AdGuardHome -RUN wget https://github.com/AdguardTeam/AdGuardHome/releases/download/v0.107.16/AdGuardHome_linux_{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}.tar.gz >/dev/null 2>&1 \ +RUN wget https://github.com/AdguardTeam/AdGuardHome/releases/download/v0.107.46/AdGuardHome_linux_{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}.tar.gz >/dev/null 2>&1 \ && mkdir -p /opt/adguardhome/conf /opt/adguardhome/work \ && tar xf AdGuardHome_linux_{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}.tar.gz ./AdGuardHome/AdGuardHome --strip-components=2 -C /opt/adguardhome \ && /bin/ash /opt/adguardhome \ diff --git a/roles/fail2ban/templates/jail.local.j2 b/roles/fail2ban/templates/jail.local.j2 index fb6cc967..1c9e0911 100755 --- a/roles/fail2ban/templates/jail.local.j2 +++ b/roles/fail2ban/templates/jail.local.j2 @@ -9,6 +9,7 @@ destemail = {{ email }} sender = {{ email }} [sshd] +backend = systemd enabled = true port = {{ ssh_port }} filter = sshd From 09643cdf7bab1b45caefa4ad73239f03df5bbd69 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Tue, 2 Apr 2024 13:10:14 +0200 Subject: [PATCH 20/48] Fix the cloudflare record deletion shell script --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7d933ed..3d8bbc83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -426,7 +426,7 @@ jobs: SERVER_NAME: ansible-easy-vpn-${{ needs.build.outputs[format('EASYVPN_USERNAME_{0}', matrix.index)] }} - name: Delete all Cloudflare domains - run: >- + run: | curl --silent "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records?per_page=50000" \ --header "Authorization: Bearer $CLOUDFLARE_TOKEN" \ | jq --raw-output '.result[].id' | while read id From e261d0e6ff9838e4bcfed3aee8af81e5f95fc93f Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Wed, 3 Apr 2024 10:34:15 +0200 Subject: [PATCH 21/48] Debug the WebUI testing --- .gitignore | 1 + testing/selenium/acceptance.py | 67 +++++++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index fb044d5b..88eb5031 100755 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ secret.yml .vscode .venv .ansible +venv* diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 598fd20a..7b89ff7c 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -49,7 +49,7 @@ def register_2fa(driver, base_url, username, password, ssh_agent): logger.debug(f"Fetching wg.{base_url}") driver.get(f"https://wg.{base_url}") - sleep(0.5) + sleep(2) logger.debug(f"Filling out the username field with {username}") username_field = driver.find_element("id", "username-textfield") username_field.send_keys(username) @@ -67,20 +67,44 @@ def register_2fa(driver, base_url, username, password, ssh_agent): register_device = driver.find_element("id", "register-link") register_device.click() + sleep(0.5) + logger.debug("Clicking on 'One-Time Password'") + one_time_password_add = driver.find_element("id", "one-time-password-add") + one_time_password_add.click() + logger.debug("Getting the notifications.txt from the server") s = pxssh.pxssh(options={"IdentityAgent": ssh_agent}) - s.login(base_url, username) - s.sendline("sudo show_2fa") + s.login(base_url, username, ssh_key="/Users/notthebee/.ssh/notthebee") + s.sendline("show_2fa") s.prompt() # Convert output to utf-8 due to pexpect weirdness notification = "\r\n".join(s.before.decode("utf-8").splitlines()[1:]) print(notification) - token = re.search("token=(.*)", notification).group(1) - driver.get(f"https://auth.{base_url}/one-time-password/register?token={token}") - sleep(2) + token = re.search("single-use code: (.*)", notification).group(1) + + one_time_password = driver.find_element("id", "one-time-code") + one_time_password.click() + sleep(0.5) + + actions = ActionChains(driver) + actions.send_keys(token) + actions.perform() + + sleep(0.5) + + verify_button = driver.find_element("id", "dialog-verify") + verify_button.click() + + sleep(3) + + next_button = driver.find_element("id", "dialog-next") + next_button.click() + + sleep(3) + secret_field = driver.find_element("id", "secret-url") secret_field = secret_field.get_attribute("value") logger.debug("Scraping the TOTP secret") @@ -91,9 +115,11 @@ def register_2fa(driver, base_url, username, password, ssh_agent): totp.now() logger.debug("Generating the OTP") - otp_done_button = driver.find_element("xpath", "//*[contains(text(), 'Done')]") - otp_done_button.click() - sleep(2) + next_button = driver.find_element("id", "dialog-next") + next_button.click() + + sleep(3) + logger.debug("Entering the OTP") actions = ActionChains(driver) @@ -102,13 +128,28 @@ def register_2fa(driver, base_url, username, password, ssh_agent): logger.debug("We're in!") sleep(1) - return + return secret -def download_wg_config(driver, base_url, client): +def download_wg_config(driver, base_url, client, secret): logger.debug(f"Opening wg.{base_url} in the browser") driver.get(f"https://wg.{base_url}") sleep(2) + + totp = pyotp.TOTP(secret) + + actions = ActionChains(driver) + actions.send_keys(totp.now()) + actions.perform() + + sleep(1) + + actions = ActionChains(driver) + actions.send_keys(totp.now()) + actions.perform() + + sleep(1) + logger.debug("Clicking on the 'New Client' button") new_client_button = driver.find_element("xpath", "//*[contains(text(), 'New Client')]") new_client_button.click() @@ -128,5 +169,5 @@ def download_wg_config(driver, base_url, client): return -register_2fa(driver, args.base_url, args.username, args.password, args.ssh_agent) -download_wg_config(driver, args.base_url, args.username) +secret = register_2fa(driver, args.base_url, args.username, args.password, args.ssh_agent) +download_wg_config(driver, args.base_url, args.username, secret) From ca571130302e6ef2dc69709c0fabc2c3ee151580 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Wed, 3 Apr 2024 10:49:57 +0200 Subject: [PATCH 22/48] Add more sleep --- testing/selenium/acceptance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 7b89ff7c..ba78745c 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -67,7 +67,7 @@ def register_2fa(driver, base_url, username, password, ssh_agent): register_device = driver.find_element("id", "register-link") register_device.click() - sleep(0.5) + sleep(2) logger.debug("Clicking on 'One-Time Password'") one_time_password_add = driver.find_element("id", "one-time-password-add") one_time_password_add.click() @@ -87,13 +87,13 @@ def register_2fa(driver, base_url, username, password, ssh_agent): one_time_password = driver.find_element("id", "one-time-code") one_time_password.click() - sleep(0.5) + sleep(1) actions = ActionChains(driver) actions.send_keys(token) actions.perform() - sleep(0.5) + sleep(1) verify_button = driver.find_element("id", "dialog-verify") verify_button.click() From f89c291eb99835b7f196bdceecb175cf8d3382d7 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Wed, 3 Apr 2024 11:07:10 +0200 Subject: [PATCH 23/48] Remove hardcoded ssh key --- testing/selenium/acceptance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index ba78745c..59335857 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -75,7 +75,7 @@ def register_2fa(driver, base_url, username, password, ssh_agent): logger.debug("Getting the notifications.txt from the server") s = pxssh.pxssh(options={"IdentityAgent": ssh_agent}) - s.login(base_url, username, ssh_key="/Users/notthebee/.ssh/notthebee") + s.login(base_url, username) s.sendline("show_2fa") s.prompt() From a64199d3fe474aca5e46d254cd1a40cbd8dc3d58 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Wed, 3 Apr 2024 19:06:19 +0200 Subject: [PATCH 24/48] Modify healthcheck for bunkerweb, fix testing script --- roles/bunkerweb/tasks/main.yml | 2 +- testing/selenium/acceptance.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/bunkerweb/tasks/main.yml b/roles/bunkerweb/tasks/main.yml index 20850564..96d42419 100755 --- a/roles/bunkerweb/tasks/main.yml +++ b/roles/bunkerweb/tasks/main.yml @@ -26,7 +26,7 @@ name: "bunkerweb" image: "bunkerity/bunkerweb:{{ bunkerweb_version }}" healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080"] + test: ["CMD", "curl", "-f", "http://localhost:80"] start_period: 10s networks: - name: wg_network diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 59335857..f9f771e6 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -148,7 +148,11 @@ def download_wg_config(driver, base_url, client, secret): actions.send_keys(totp.now()) actions.perform() - sleep(1) + sleep(2) + + logger.debug(f"Opening wg.{base_url} in the browser") + driver.get(f"https://wg.{base_url}") + sleep(2) logger.debug("Clicking on the 'New Client' button") new_client_button = driver.find_element("xpath", "//*[contains(text(), 'New Client')]") From c706ec23158105a8951f1d9d45c9f27f54ac1a88 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Wed, 3 Apr 2024 19:22:09 +0200 Subject: [PATCH 25/48] Fix tests, attempt to fix fail2ban --- roles/fail2ban/templates/jail.local.j2 | 24 ++++++++++++------------ testing/selenium/acceptance.py | 5 +++-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/roles/fail2ban/templates/jail.local.j2 b/roles/fail2ban/templates/jail.local.j2 index 1c9e0911..0d7fb2bb 100755 --- a/roles/fail2ban/templates/jail.local.j2 +++ b/roles/fail2ban/templates/jail.local.j2 @@ -1,15 +1,15 @@ [DEFAULT] -banaction = iptables-allports -bantime = -1 -findtime = 600 -maxretry = 5 -ignoreip = 127.0.0.1/8 ::1 -action = %(action_mwl)s -destemail = {{ email }} -sender = {{ email }} +banaction=iptables-allports +bantime=-1 +findtime=600 +maxretry=5 +ignoreip=127.0.0.1/8 ::1 +action=%(action_mwl)s +destemail={{ email }} +sender={{ email }} [sshd] -backend = systemd -enabled = true -port = {{ ssh_port }} -filter = sshd +backend=systemd +enabled=true +port={{ ssh_port }} +filter=sshd diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index f9f771e6..248105a6 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -144,15 +144,16 @@ def download_wg_config(driver, base_url, client, secret): sleep(1) + # Sometimes Authelia does not accept the TOTP code the first time... actions = ActionChains(driver) actions.send_keys(totp.now()) actions.perform() - sleep(2) + sleep(1) logger.debug(f"Opening wg.{base_url} in the browser") driver.get(f"https://wg.{base_url}") - sleep(2) + sleep(5) logger.debug("Clicking on the 'New Client' button") new_client_button = driver.find_element("xpath", "//*[contains(text(), 'New Client')]") From da8316c5bfd1e08a294b1ae689ea7f183bb881fa Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Wed, 3 Apr 2024 19:29:19 +0200 Subject: [PATCH 26/48] Debug fail2ban --- roles/fail2ban/tasks/main.yml | 9 +++++++++ roles/system/handlers/main.yml | 5 +++++ roles/system/tasks/firewall.yml | 1 + 3 files changed, 15 insertions(+) create mode 100644 roles/system/handlers/main.yml diff --git a/roles/fail2ban/tasks/main.yml b/roles/fail2ban/tasks/main.yml index bae462a5..3efa3bae 100755 --- a/roles/fail2ban/tasks/main.yml +++ b/roles/fail2ban/tasks/main.yml @@ -29,3 +29,12 @@ name: fail2ban state: started enabled: yes + +- name: Check the fail2ban service status + register: fail2ban_service + ansible.builtin.systemd_service: + name: fail2ban + +- name: Fail if fail2ban failed + ansible.builtin.assert: + that: fail2ban_service.StatusErrno != "0" diff --git a/roles/system/handlers/main.yml b/roles/system/handlers/main.yml new file mode 100644 index 00000000..c990df82 --- /dev/null +++ b/roles/system/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart iptables + ansible.builtin.service: + name: iptables + state: restarted diff --git a/roles/system/tasks/firewall.yml b/roles/system/tasks/firewall.yml index f9bbf197..6d164f95 100644 --- a/roles/system/tasks/firewall.yml +++ b/roles/system/tasks/firewall.yml @@ -15,6 +15,7 @@ cmd: "which iptables-restore" - name: Template the iptables systemd service + notify: Restart iptables ansible.builtin.template: src: systemd/iptables.service.j2 dest: /etc/systemd/system/iptables.service From 7b0ac8e05429d1d032f9ac5e48c98202c2ee8e4d Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Wed, 3 Apr 2024 19:39:25 +0200 Subject: [PATCH 27/48] Debug fail2ban --- roles/fail2ban/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/fail2ban/tasks/main.yml b/roles/fail2ban/tasks/main.yml index 3efa3bae..d421326c 100755 --- a/roles/fail2ban/tasks/main.yml +++ b/roles/fail2ban/tasks/main.yml @@ -37,4 +37,4 @@ - name: Fail if fail2ban failed ansible.builtin.assert: - that: fail2ban_service.StatusErrno != "0" + that: fail2ban_service.status.StatusErrno != "0" From 63059e9fbc52660afa45f5ad2302000f27bd19d8 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 09:32:51 +0200 Subject: [PATCH 28/48] Modify debug step for GH actions --- .github/workflows/ci.yml | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d8bbc83..b7e343e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,10 +249,6 @@ jobs: env: LETSENCRYPT_STAGING: ${{ needs.matrix_prep.outputs.letsencrypt_staging }} - - name: Sleep forever - run: sleep infinity - if: inputs.manual_mode - - name: Archive the private SSH key (Matrix 1) if: ${{ matrix.index == '1' }} uses: actions/upload-artifact@v3 @@ -280,15 +276,6 @@ jobs: with: name: "private-ssh-key-4" path: "id_vpn" - - - - - - - - - outputs: EASYVPN_USERNAME_1: "${{ steps.random_username.outputs.EASYVPN_USERNAME_1 }}" EASYVPN_USERNAME_2: "${{ steps.random_username.outputs.EASYVPN_USERNAME_2 }}" @@ -299,6 +286,23 @@ jobs: EASYVPN_PASSWORD_3: "${{ steps.random_username.outputs.EASYVPN_PASSWORD_3 }}" EASYVPN_PASSWORD_4: "${{ steps.random_username.outputs.EASYVPN_PASSWORD_4 }}" + + debug: + runs-on: ubuntu-latest + environment: cicd + if: always() + strategy: + fail-fast: false + matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} + needs: + - matrix_prep + - build + steps: + - name: Wait for the server to reboot + if: inputs.manual_mode + run: >- + sleep infinity + fetch_config: runs-on: ubuntu-latest environment: cicd @@ -353,8 +357,6 @@ jobs: name: "private-ssh-key-4" path: /home/runner/.ssh - - - name: Set the correct permissions for the SSH key run: | chmod 700 $HOME/.ssh From 1bac250dda43d3a513fdf4cd9315d713e8f4ab6f Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 11:07:37 +0200 Subject: [PATCH 29/48] Modify debug step for GH actions --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7e343e8..269191f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -414,6 +414,7 @@ jobs: needs: - matrix_prep - build + - debug - fetch_config steps: - name: Destroy the Hetzner instances From 6561fa9ae70c3cf048f3bd520ba2c803f028619f Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 19:49:44 +0200 Subject: [PATCH 30/48] Debug fail2ban --- .github/workflows/ci.yml | 1 - roles/fail2ban/tasks/main.yml | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 269191f2..0963b94f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -290,7 +290,6 @@ jobs: debug: runs-on: ubuntu-latest environment: cicd - if: always() strategy: fail-fast: false matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} diff --git a/roles/fail2ban/tasks/main.yml b/roles/fail2ban/tasks/main.yml index d421326c..8ae8dea9 100755 --- a/roles/fail2ban/tasks/main.yml +++ b/roles/fail2ban/tasks/main.yml @@ -36,5 +36,7 @@ name: fail2ban - name: Fail if fail2ban failed + failed_when: fail2ban_failed and not letsencrypt_staging + register: fail2ban_failed ansible.builtin.assert: that: fail2ban_service.status.StatusErrno != "0" From e82e7847f03235d3a1c28d8213d19b670609ec08 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 20:06:33 +0200 Subject: [PATCH 31/48] Debug fail2ban --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0963b94f..bf0a4570 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -227,8 +227,6 @@ jobs: with: ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Regsiter the private key with the ssh-agent run: expect -c "spawn ssh-add $HOME/.ssh/id_vpn; expect -re \"Enter passphrase.*\"; send -- \"$EASYVPN_PASSWORD\r\"; expect -re \"Identity added.*\"" @@ -249,6 +247,10 @@ jobs: env: LETSENCRYPT_STAGING: ${{ needs.matrix_prep.outputs.letsencrypt_staging }} + - name: Add the personal SSH key for debugging + run: >- + ssh -t -t $EASYVPN_USERNAME@$SERVER_IPV$ "curl https://github.com/notthebee.keys >> $HOME/.ssh/authorized_keys" + - name: Archive the private SSH key (Matrix 1) if: ${{ matrix.index == '1' }} uses: actions/upload-artifact@v3 @@ -310,6 +312,7 @@ jobs: matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} needs: - matrix_prep + - debug - build steps: - name: Check out this repo From 42734bdeb07957603fec0ae2b49b269547294f6d Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 20:17:26 +0200 Subject: [PATCH 32/48] [skip ci] Debug fail2ban --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf0a4570..bc3b3108 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,7 +249,7 @@ jobs: - name: Add the personal SSH key for debugging run: >- - ssh -t -t $EASYVPN_USERNAME@$SERVER_IPV$ "curl https://github.com/notthebee.keys >> $HOME/.ssh/authorized_keys" + ssh -t -t $EASYVPN_USERNAME@$SERVER_IPV4 "curl https://github.com/notthebee.keys >> $HOME/.ssh/authorized_keys" - name: Archive the private SSH key (Matrix 1) if: ${{ matrix.index == '1' }} From 4bdb19c298f4c0c03f4ed37c9e942cdbe48afef6 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 20:31:14 +0200 Subject: [PATCH 33/48] [skip ci] Debug fail2ban --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc3b3108..18400f83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,7 +249,7 @@ jobs: - name: Add the personal SSH key for debugging run: >- - ssh -t -t $EASYVPN_USERNAME@$SERVER_IPV4 "curl https://github.com/notthebee.keys >> $HOME/.ssh/authorized_keys" + ssh -t -t $EASYVPN_USERNAME@$SERVER_IPV4 "curl https://github.com/notthebee.keys >> /home/$EASYVPN_USERNAME/.ssh/authorized_keys" - name: Archive the private SSH key (Matrix 1) if: ${{ matrix.index == '1' }} From dc9b5564b2236e4c5938b6e08d9b20df7c73e963 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 20:53:28 +0200 Subject: [PATCH 34/48] Add missing fail2ban dependency --- roles/fail2ban/tasks/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/fail2ban/tasks/main.yml b/roles/fail2ban/tasks/main.yml index 8ae8dea9..4e8e5241 100755 --- a/roles/fail2ban/tasks/main.yml +++ b/roles/fail2ban/tasks/main.yml @@ -1,8 +1,9 @@ --- - name: Install fail2ban - ansible.builtin.package: - name: fail2ban - state: present + ansible.builtin.apt: + name: + - fail2ban + - python3-systemd - name: Disable e-mail notifications on jail stop and start ansible.builtin.copy: From c38928d56476b799128a5c50d20d0682c3f32fd5 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 21:28:37 +0200 Subject: [PATCH 35/48] Remove f2b checks --- roles/fail2ban/tasks/main.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/roles/fail2ban/tasks/main.yml b/roles/fail2ban/tasks/main.yml index 4e8e5241..dc917e46 100755 --- a/roles/fail2ban/tasks/main.yml +++ b/roles/fail2ban/tasks/main.yml @@ -30,14 +30,3 @@ name: fail2ban state: started enabled: yes - -- name: Check the fail2ban service status - register: fail2ban_service - ansible.builtin.systemd_service: - name: fail2ban - -- name: Fail if fail2ban failed - failed_when: fail2ban_failed and not letsencrypt_staging - register: fail2ban_failed - ansible.builtin.assert: - that: fail2ban_service.status.StatusErrno != "0" From 33c9b2e37358b5b789fb48af8b6f711832f9bea5 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 22:52:06 +0200 Subject: [PATCH 36/48] Debug ci/cd testing --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18400f83..13c4c456 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,7 +248,8 @@ jobs: LETSENCRYPT_STAGING: ${{ needs.matrix_prep.outputs.letsencrypt_staging }} - name: Add the personal SSH key for debugging - run: >- + run: | + ssh -t -t $EASYVPN_USERNAME@$SERVER_IPV4 "echo '\n' >> /home/$EASYVPN_USERNAME/.ssh/authorized_keys" ssh -t -t $EASYVPN_USERNAME@$SERVER_IPV4 "curl https://github.com/notthebee.keys >> /home/$EASYVPN_USERNAME/.ssh/authorized_keys" - name: Archive the private SSH key (Matrix 1) From 5db3a2040e7b6b82f58338c2646af3277beb7079 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Thu, 4 Apr 2024 23:01:52 +0200 Subject: [PATCH 37/48] Fix adguard Dockerfile template --- roles/dns/files/adguard-unbound/Dockerfile | 34 ---------------------- roles/dns/tasks/main.yml | 10 ++++++- 2 files changed, 9 insertions(+), 35 deletions(-) delete mode 100644 roles/dns/files/adguard-unbound/Dockerfile diff --git a/roles/dns/files/adguard-unbound/Dockerfile b/roles/dns/files/adguard-unbound/Dockerfile deleted file mode 100644 index 0c74c18e..00000000 --- a/roles/dns/files/adguard-unbound/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM alpine:3.15 - -RUN apk add --no-cache \ - libcap \ - unbound=1.13.2-r2 \ - dnscrypt-proxy - -WORKDIR /tmp - -RUN wget https://www.internic.net/domain/named.root -qO- >> /etc/unbound/root.hints - -COPY files/ /opt/ - -# AdGuardHome -RUN wget https://github.com/AdguardTeam/AdGuardHome/releases/download/v0.107.16/AdGuardHome_linux_amd64.tar.gz >/dev/null 2>&1 \ - && mkdir -p /opt/adguardhome/conf /opt/adguardhome/work \ - && tar xf AdGuardHome_linux_amd64.tar.gz ./AdGuardHome/AdGuardHome --strip-components=2 -C /opt/adguardhome \ - && /bin/ash /opt/adguardhome \ - && chown -R nobody: /opt/adguardhome \ - && setcap 'CAP_NET_BIND_SERVICE=+eip CAP_NET_RAW=+eip' /opt/adguardhome/AdGuardHome \ - && chmod +x /opt/entrypoint.sh \ - && rm -rf /tmp/* /var/cache/apk/* - -WORKDIR /opt/adguardhome/work - -VOLUME ["/opt/adguardhome/conf", "/opt/adguardhome/work", "/opt/unbound"] - -EXPOSE 53/tcp 53/udp 67/udp 68/udp 80/tcp 443/tcp 853/tcp 3000/tcp 5053/udp 5053/tcp - -HEALTHCHECK --interval=30s --timeout=15s --start-period=5s\ - CMD sh /opt/healthcheck.sh - -CMD ["/opt/entrypoint.sh"] - diff --git a/roles/dns/tasks/main.yml b/roles/dns/tasks/main.yml index b204cf99..4b5ff61f 100644 --- a/roles/dns/tasks/main.yml +++ b/roles/dns/tasks/main.yml @@ -5,7 +5,7 @@ state: directory owner: "{{ username }}" group: "{{ username }}" - mode: 0755 + mode: "0755" loop: - adguard-unbound-doh - adguard-unbound-doh/adguard @@ -22,6 +22,14 @@ rsync_opts: - "--ignore-existing" +- name: Template the Adguard Dockerfile + ansible.builtin.template: + src: Dockerfile + dest: "{{ docker_dir }}/adguard-unbound-doh/build/Dockerfile" + owner: "{{ username }}" + group: "{{ username }}" + mode: "0775" + - name: Build the adguard-unbound Docker image community.docker.docker_image: name: adguard-unbound-doh From f4b5e10f1f1dde6fe84c4889451700eb632c46c3 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 12:45:35 +0200 Subject: [PATCH 38/48] Fix personal SSH key --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13c4c456..66bcab3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,9 +248,9 @@ jobs: LETSENCRYPT_STAGING: ${{ needs.matrix_prep.outputs.letsencrypt_staging }} - name: Add the personal SSH key for debugging - run: | - ssh -t -t $EASYVPN_USERNAME@$SERVER_IPV4 "echo '\n' >> /home/$EASYVPN_USERNAME/.ssh/authorized_keys" - ssh -t -t $EASYVPN_USERNAME@$SERVER_IPV4 "curl https://github.com/notthebee.keys >> /home/$EASYVPN_USERNAME/.ssh/authorized_keys" + run: >- + curl https://github.com/notthebee.keys > notthebee.pub && + ssh-copy-id -f -i notthebee.pub $EASYVPN_USERNAME@$SERVER_IPV4 - name: Archive the private SSH key (Matrix 1) if: ${{ matrix.index == '1' }} From 5d674d0619d2ebd4eca6ed108304ca71bd9c91ee Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 13:45:38 +0200 Subject: [PATCH 39/48] Modify testing --- .github/workflows/ci.yml | 60 ++++------------- testing/selenium/acceptance.py | 116 ++++++++++++++++----------------- 2 files changed, 68 insertions(+), 108 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66bcab3d..0616d98f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,32 +253,11 @@ jobs: ssh-copy-id -f -i notthebee.pub $EASYVPN_USERNAME@$SERVER_IPV4 - name: Archive the private SSH key (Matrix 1) - if: ${{ matrix.index == '1' }} uses: actions/upload-artifact@v3 with: - name: "private-ssh-key-1" + name: "private-ssh-key-{{ matrix.index }}" path: "id_vpn" - - name: Archive the private SSH key (Matrix 2) - if: ${{ matrix.index == '2' }} - uses: actions/upload-artifact@v3 - with: - name: "private-ssh-key-2" - path: "id_vpn" - - - name: Archive the private SSH key (Matrix 3) - if: ${{ matrix.index == '3' }} - uses: actions/upload-artifact@v3 - with: - name: "private-ssh-key-3" - path: "id_vpn" - - - name: Archive the private SSH key (Matrix 4) - if: ${{ matrix.index == '4' }} - uses: actions/upload-artifact@v3 - with: - name: "private-ssh-key-4" - path: "id_vpn" outputs: EASYVPN_USERNAME_1: "${{ steps.random_username.outputs.EASYVPN_USERNAME_1 }}" EASYVPN_USERNAME_2: "${{ steps.random_username.outputs.EASYVPN_USERNAME_2 }}" @@ -332,32 +311,10 @@ jobs: run: >- mkdir /home/runner/.ssh - - name: Get the private SSH key artifact (Matrix 1) - if: matrix.index == '1' - uses: actions/download-artifact@v3 - with: - name: "private-ssh-key-1" - path: /home/runner/.ssh - - - name: Get the private SSH key artifact (Matrix 2) - if: matrix.index == '2' - uses: actions/download-artifact@v3 - with: - name: "private-ssh-key-2" - path: /home/runner/.ssh - - - name: Get the private SSH key artifact (Matrix 3) - if: matrix.index == '3' - uses: actions/download-artifact@v3 - with: - name: "private-ssh-key-3" - path: /home/runner/.ssh - - - name: Get the private SSH key artifact (Matrix 4) - if: matrix.index == '4' + - name: Get the private SSH key artifact uses: actions/download-artifact@v3 with: - name: "private-ssh-key-4" + name: "private-ssh-key-{{ matrix.index }}" path: /home/runner/.ssh - name: Set the correct permissions for the SSH key @@ -404,8 +361,15 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: "Screenshots" - path: "screenshots/" + name: "Screenshots-{{ matrix.index }}" + path: "screenshots/*.png" + + - name: Archive the Wireguard config (Matrix 3) + if: ${{ matrix.index == '3' }} + uses: actions/upload-artifact@v3 + with: + name: "wireguard-{{ matrix.index }}.conf" + path: "*.conf" destroy: runs-on: ubuntu-latest diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 248105a6..2fc2f40e 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -2,8 +2,12 @@ from pexpect import pxssh from selenium import webdriver +from selenium.common.exceptions import TimeoutException from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.common.by import By from selenium.webdriver.common.action_chains import ActionChains from time import sleep @@ -14,7 +18,8 @@ import re -service = Service(executable_path=r"/snap/bin/chromium.chromedriver") +service = Service(executable_path=r"/opt/homebrew/bin/chromedriver") +#service = Service(executable_path=r"/snap/bin/chromium.chromedriver") parser = argparse.ArgumentParser() parser.add_argument("--username", type=str, metavar="username") @@ -26,9 +31,10 @@ chrome_options = Options() prefs = {"download.default_directory": "/home/runner"} +#prefs = {"download.default_directory": "/Users/notthebee/Downloads"} chrome_options.add_experimental_option("prefs", prefs) options = [ - "--headless", + #"--headless", "--disable-gpu", "--window-size=1920,1200", "--ignore-certificate-errors", @@ -46,64 +52,57 @@ logger.setLevel(logging.DEBUG) +def save_screenshot(screenshot_name): + screenshot_path = "/home/runner/screenshots/" + #screenshot_path = "/Users/notthebee/Downloads/" + driver.save_screenshot(screenshot_path + screenshot_name) + return + def register_2fa(driver, base_url, username, password, ssh_agent): logger.debug(f"Fetching wg.{base_url}") driver.get(f"https://wg.{base_url}") - sleep(2) + logger.debug(f"Filling out the username field with {username}") - username_field = driver.find_element("id", "username-textfield") - username_field.send_keys(username) - sleep(0.5) + WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "username-textfield"))).send_keys(username) + save_screenshot("1_AutheliaUsername.png") logger.debug(f"Filling out the password field with {password}") - password_field = driver.find_element("id", "password-textfield") - password_field.send_keys(password) - sleep(0.5) + WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "password-textfield"))).send_keys(password) + save_screenshot("2_AutheliaPassword.png") + logger.debug("Signing in...") - submit_button = driver.find_element("id", "sign-in-button") - submit_button.click() - sleep(5) + WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "sign-in-button"))).click() logger.debug("Clicking on 'Register device'") - register_device = driver.find_element("id", "register-link") - register_device.click() + WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "register-link"))).click() + save_screenshot("3_RegisterDevice.png") - sleep(2) logger.debug("Clicking on 'One-Time Password'") - one_time_password_add = driver.find_element("id", "one-time-password-add") - one_time_password_add.click() + WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "one-time-password-add"))).click() + save_screenshot("4_OTP.png") logger.debug("Getting the notifications.txt from the server") - s = pxssh.pxssh(options={"IdentityAgent": ssh_agent}) s.login(base_url, username) s.sendline("show_2fa") s.prompt() - # Convert output to utf-8 due to pexpect weirdness notification = "\r\n".join(s.before.decode("utf-8").splitlines()[1:]) print(notification) - token = re.search("single-use code: (.*)", notification).group(1) - one_time_password = driver.find_element("id", "one-time-code") - one_time_password.click() - sleep(1) + logger.debug("Entering the one-time code") + WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "one-time-code"))).click() + save_screenshot("5_EnteringOTP.png") actions = ActionChains(driver) actions.send_keys(token) actions.perform() - sleep(1) - - verify_button = driver.find_element("id", "dialog-verify") - verify_button.click() - - sleep(3) + WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-verify"))).click() + WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-next"))).click() - next_button = driver.find_element("id", "dialog-next") - next_button.click() - - sleep(3) + WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "secret-url"))) + save_screenshot("6_SecetURL.png") secret_field = driver.find_element("id", "secret-url") secret_field = secret_field.get_attribute("value") @@ -115,8 +114,7 @@ def register_2fa(driver, base_url, username, password, ssh_agent): totp.now() logger.debug("Generating the OTP") - next_button = driver.find_element("id", "dialog-next") - next_button.click() + WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-next"))).click() sleep(3) @@ -134,42 +132,40 @@ def register_2fa(driver, base_url, username, password, ssh_agent): def download_wg_config(driver, base_url, client, secret): logger.debug(f"Opening wg.{base_url} in the browser") driver.get(f"https://wg.{base_url}") - sleep(2) + save_screenshot("7_AutheliaOTPEnter.png") totp = pyotp.TOTP(secret) - actions = ActionChains(driver) - actions.send_keys(totp.now()) - actions.perform() - sleep(1) + logger.debug("Clicking on the 'New Client' button") - # Sometimes Authelia does not accept the TOTP code the first time... - actions = ActionChains(driver) - actions.send_keys(totp.now()) - actions.perform() + attempts = 0 + while attempts < 5: + try: + actions = ActionChains(driver) + actions.send_keys(totp.now()) + actions.perform() - sleep(1) + logger.debug(f"Opening wg.{base_url} in the browser") + driver.get(f"https://wg.{base_url}") - logger.debug(f"Opening wg.{base_url} in the browser") - driver.get(f"https://wg.{base_url}") - sleep(5) + WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, "//*[contains(text(), 'New Client')]"))).click() + save_screenshot("8_WGEeasy_NewClient.png") + break + except TimeoutException: + attempts += 1 - logger.debug("Clicking on the 'New Client' button") - new_client_button = driver.find_element("xpath", "//*[contains(text(), 'New Client')]") - new_client_button.click() - sleep(2) logger.debug(f"Filling out the 'Name' field with {client}") - name_field = driver.find_element("xpath", "//input[@placeholder='Name']") - name_field.send_keys(client) - sleep(2) + WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//input[@placeholder='Name']"))).send_keys(client) + save_screenshot("9_WGEeasy_NewClientName.png") + logger.debug("Clicking on 'Create'") - create_button = driver.find_element("xpath", "//*[contains(text(), 'Create')]") - create_button.click() - sleep(2) + WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[contains(text(), 'Create')]"))).click() + + save_screenshot("10_WGEeasy_ClientCreated.png") logger.debug("Downloading the configuration") - download_config = driver.find_element("xpath", "//a[@title='Download Configuration']") - download_config.click() + WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[@title='Download Configuration']"))).click() + sleep(2) return From 1811b2e3bc0c0f7c4080f073bd36e810222acb4c Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 14:02:20 +0200 Subject: [PATCH 40/48] Fixup --- testing/selenium/acceptance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 2fc2f40e..a5154726 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -18,8 +18,8 @@ import re -service = Service(executable_path=r"/opt/homebrew/bin/chromedriver") -#service = Service(executable_path=r"/snap/bin/chromium.chromedriver") +#service = Service(executable_path=r"/opt/homebrew/bin/chromedriver") +service = Service(executable_path=r"/snap/bin/chromium.chromedriver") parser = argparse.ArgumentParser() parser.add_argument("--username", type=str, metavar="username") From 3fdd71e1ec70f9c091cd35a78a29bbc4f13c513f Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 14:38:25 +0200 Subject: [PATCH 41/48] Fixup --- testing/selenium/acceptance.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index a5154726..51d74d8b 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -18,7 +18,6 @@ import re -#service = Service(executable_path=r"/opt/homebrew/bin/chromedriver") service = Service(executable_path=r"/snap/bin/chromium.chromedriver") parser = argparse.ArgumentParser() @@ -31,10 +30,9 @@ chrome_options = Options() prefs = {"download.default_directory": "/home/runner"} -#prefs = {"download.default_directory": "/Users/notthebee/Downloads"} chrome_options.add_experimental_option("prefs", prefs) options = [ - #"--headless", + "--headless", "--disable-gpu", "--window-size=1920,1200", "--ignore-certificate-errors", From 5d792578703c84f60f9c96c9dff492c56da2a9d1 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 15:02:53 +0200 Subject: [PATCH 42/48] Fixup --- .github/workflows/ci.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0616d98f..758878a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,10 +252,10 @@ jobs: curl https://github.com/notthebee.keys > notthebee.pub && ssh-copy-id -f -i notthebee.pub $EASYVPN_USERNAME@$SERVER_IPV4 - - name: Archive the private SSH key (Matrix 1) + - name: Archive the private SSH key uses: actions/upload-artifact@v3 with: - name: "private-ssh-key-{{ matrix.index }}" + name: "private-ssh-key-${{ matrix.index }}" path: "id_vpn" outputs: @@ -314,7 +314,7 @@ jobs: - name: Get the private SSH key artifact uses: actions/download-artifact@v3 with: - name: "private-ssh-key-{{ matrix.index }}" + name: "private-ssh-key-${{ matrix.index }}" path: /home/runner/.ssh - name: Set the correct permissions for the SSH key @@ -361,14 +361,13 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: "Screenshots-{{ matrix.index }}" - path: "screenshots/*.png" + name: "Screenshots-${{ matrix.index }}" + path: "screenshots" - - name: Archive the Wireguard config (Matrix 3) - if: ${{ matrix.index == '3' }} + - name: Archive the Wireguard config uses: actions/upload-artifact@v3 with: - name: "wireguard-{{ matrix.index }}.conf" + name: "wireguard-${{ matrix.index }}.conf" path: "*.conf" destroy: From 3a2fbcd8d579384eda0c3aa1fbd272e8de2bc05f Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 15:17:31 +0200 Subject: [PATCH 43/48] Fix screenshot path --- .github/workflows/ci.yml | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 758878a1..049f59b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,27 +118,8 @@ jobs: - name: Set the username and password outputs id: random_username run: | - case $INDEX in - "1") - echo "EASYVPN_USERNAME_1=$EASYVPN_USERNAME" >> $GITHUB_OUTPUT - echo "EASYVPN_PASSWORD_1=$EASYVPN_PASSWORD" >> $GITHUB_OUTPUT - ;; - "2") - echo "EASYVPN_USERNAME_2=$EASYVPN_USERNAME" >> $GITHUB_OUTPUT - echo "EASYVPN_PASSWORD_2=$EASYVPN_PASSWORD" >> $GITHUB_OUTPUT - ;; - "3") - echo "EASYVPN_USERNAME_3=$EASYVPN_USERNAME" >> $GITHUB_OUTPUT - echo "EASYVPN_PASSWORD_3=$EASYVPN_PASSWORD" >> $GITHUB_OUTPUT - ;; - "4") - echo "EASYVPN_USERNAME_4=$EASYVPN_USERNAME" >> $GITHUB_OUTPUT - echo "EASYVPN_PASSWORD_4=$EASYVPN_PASSWORD" >> $GITHUB_OUTPUT - ;; - *) - exit 1 - ;; - esac + echo "EASYVPN_USERNAME_$INDEX=$EASYVPN_USERNAME" >> $GITHUB_OUTPUT + echo "EASYVPN_PASSWORD_$INDEX=$EASYVPN_PASSWORD" >> $GITHUB_OUTPUT env: INDEX: ${{ matrix.index }} @@ -362,7 +343,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: "Screenshots-${{ matrix.index }}" - path: "screenshots" + path: "/home/runner/screenshots" - name: Archive the Wireguard config uses: actions/upload-artifact@v3 From 76594b4348eeb6d24cf0e2b970ad503d2abd8fe5 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 15:20:20 +0200 Subject: [PATCH 44/48] Try to debug missing secret field on ubuntu --- testing/selenium/acceptance.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 51d74d8b..c2dde624 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -99,11 +99,9 @@ def register_2fa(driver, base_url, username, password, ssh_agent): WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-verify"))).click() WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-next"))).click() - WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "secret-url"))) + WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "secret-url"))).get_attribute("value") save_screenshot("6_SecetURL.png") - secret_field = driver.find_element("id", "secret-url") - secret_field = secret_field.get_attribute("value") logger.debug("Scraping the TOTP secret") secret = re.search("secret=(.*)", secret_field).group(1) From b9ffcbe2687b0defb68d27f47f37c7ce0e146b85 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 15:33:29 +0200 Subject: [PATCH 45/48] Fixup --- testing/selenium/acceptance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index c2dde624..56dd6c3e 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -99,7 +99,7 @@ def register_2fa(driver, base_url, username, password, ssh_agent): WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-verify"))).click() WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-next"))).click() - WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "secret-url"))).get_attribute("value") + secret_field = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "secret-url"))).get_attribute("value") save_screenshot("6_SecetURL.png") logger.debug("Scraping the TOTP secret") From ed5e3a8025769d4568620df3ef583cdb336da152 Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 15:56:12 +0200 Subject: [PATCH 46/48] Debug the secret uri again... --- .github/workflows/ci.yml | 2 +- testing/selenium/acceptance.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 049f59b4..f60beffa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -343,7 +343,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: "Screenshots-${{ matrix.index }}" - path: "/home/runner/screenshots" + path: "/home/runner/screenshots/*" - name: Archive the Wireguard config uses: actions/upload-artifact@v3 diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 56dd6c3e..164c1c29 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -52,7 +52,6 @@ def save_screenshot(screenshot_name): screenshot_path = "/home/runner/screenshots/" - #screenshot_path = "/Users/notthebee/Downloads/" driver.save_screenshot(screenshot_path + screenshot_name) return @@ -99,13 +98,18 @@ def register_2fa(driver, base_url, username, password, ssh_agent): WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-verify"))).click() WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-next"))).click() - secret_field = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "secret-url"))).get_attribute("value") - save_screenshot("6_SecetURL.png") - logger.debug("Scraping the TOTP secret") - secret = re.search("secret=(.*)", secret_field).group(1) + attempts = 0 + while attempts < 5: + try: + secret_field = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "secret-url"))).get_attribute("value") + secret = re.search("secret=(.*)", secret_field).group(1) + break + except AttributeError: + attempts += 1 + save_screenshot("6_SecetURL.png") totp = pyotp.TOTP(secret) totp.now() logger.debug("Generating the OTP") From e9d6c6da5079ba0e6c6c3e0363419437028cee0b Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 16:17:15 +0200 Subject: [PATCH 47/48] Set DNS to Cloudflare during Docker build --- roles/dns/templates/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/dns/templates/Dockerfile b/roles/dns/templates/Dockerfile index 0557dfea..6ceacb63 100644 --- a/roles/dns/templates/Dockerfile +++ b/roles/dns/templates/Dockerfile @@ -8,7 +8,8 @@ RUN apk add --no-cache \ WORKDIR /tmp -RUN wget https://www.internic.net/domain/named.root -qO- >> /etc/unbound/root.hints +RUN echo "nameserver 1.1.1.1" > /etc/resolv.conf && \ + wget https://www.internic.net/domain/named.root -qO- >> /etc/unbound/root.hints COPY files/ /opt/ From cec942a9a7a04612aceada0118d9183dfa52c6fd Mon Sep 17 00:00:00 2001 From: Wolfgang Date: Fri, 5 Apr 2024 16:59:55 +0200 Subject: [PATCH 48/48] Debug testing --- testing/selenium/acceptance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/selenium/acceptance.py b/testing/selenium/acceptance.py index 164c1c29..0776ab13 100755 --- a/testing/selenium/acceptance.py +++ b/testing/selenium/acceptance.py @@ -98,11 +98,12 @@ def register_2fa(driver, base_url, username, password, ssh_agent): WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-verify"))).click() WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "dialog-next"))).click() - logger.debug("Scraping the TOTP secret") + secret = "" attempts = 0 while attempts < 5: try: + logger.debug("Scraping the TOTP secret") secret_field = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "secret-url"))).get_attribute("value") secret = re.search("secret=(.*)", secret_field).group(1) break