From d2ff667177a63e30eb9ff4afa842cae629f86d28 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Wed, 28 Dec 2022 16:07:51 -0300 Subject: [PATCH] upstream CI multihost: Create multihost environment To be able to use a multihost testing environment, it is required that a domain with a server along with clients or replcas and clients be deployed. This environment is much more complex to obtain using the current CI infrastructure that uses molecule and containers, due to the need of more isolation between the testing nodes. By using Github actions, and Github's hosted macOS runner along with Vagrant to spawn multiple virtual hosts, it is possible to create an environment with a few virtual machines, that provided the required isolation. This setup allows for both deployment role testing, and multihost testing. The runner has to be a macOS runner due to Github restrictions on nested virtualization. The runner has support for Python 3, and the latest versio of ansible-core is installed through 'pip. This host has a 3-core vCPU, 14 GB of RAM and 14Gb of storage. The guests configuration are: * server.ipa.test: 2500 MB of RAM * rep-01.ipa.test: 2500 MB of RAM * cli-01.ipa.test: 768 MB of RAM All guests are deployed with the oficial Fedora 38 cloud-base image. Workflow steps are executed from '/tests/multihost' unless this is overriden with 'working-directory'. As Github sets the proper working directory only when 'run' is executed, the default directory is the repository root (e.g. setting 'working-directory: .' will set the working directory to the repository root). Although it is possible to change the working-directory, a different configuration has not been tested. The playbooks were created so that environment variables can be used to change the domain configuration. This can be used to create multiple parallel jobs in a test matrix. The default configuration installs a server with embedded DNS, a replica with no extra service, and a client. --- .ansible-lint | 1 + .github/workflows/multihost.yml | 93 +++++++++++++++++++ .gitignore | 4 + tests/multihost/.gitignore | 2 + tests/multihost/README-vagrant.md | 93 +++++++++++++++++++ tests/multihost/Vagrantfile | 55 +++++++++++ tests/multihost/ensure-reverse-dns.yaml | 18 ++++ tests/multihost/get_ip.sh | 13 +++ tests/multihost/inventory/group_vars/all.yml | 2 + .../multihost/inventory/vagrant-inventory.yml | 89 ++++++++++++++++++ tests/multihost/playbooks | 1 + 11 files changed, 371 insertions(+) create mode 100644 .github/workflows/multihost.yml create mode 100644 tests/multihost/.gitignore create mode 100644 tests/multihost/README-vagrant.md create mode 100644 tests/multihost/Vagrantfile create mode 100644 tests/multihost/ensure-reverse-dns.yaml create mode 100755 tests/multihost/get_ip.sh create mode 100644 tests/multihost/inventory/group_vars/all.yml create mode 100644 tests/multihost/inventory/vagrant-inventory.yml create mode 120000 tests/multihost/playbooks diff --git a/.ansible-lint b/.ansible-lint index abb18e9275..fd0dee3349 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -12,6 +12,7 @@ exclude_paths: - meta/runtime.yml - requirements-docker.yml - requirements-podman.yml + - tests/multihost/vagrant-inventory.yml kinds: - playbook: '**/tests/**/test_*.yml' diff --git a/.github/workflows/multihost.yml b/.github/workflows/multihost.yml new file mode 100644 index 0000000000..a36407e103 --- /dev/null +++ b/.github/workflows/multihost.yml @@ -0,0 +1,93 @@ +name: Multihost Testing + +on: + - push + - pull_request + +jobs: + multihost-testing: + name: "Multihost tests" + # Only macos provides Vagrant. + runs-on: macos-12 + defaults: + run: + working-directory: tests/multihost + + steps: + - uses: actions/checkout@v3.1.0 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v4.3.0 + with: + python-version: "3.x" + + - name: Install Ansible + run: pip install ansible-core + + - name: Ansible version + run: ansible --version + + - name: Prepare ansible-freeipa environment + working-directory: . + run: | + rm -rf ~/.ansible + mkdir ~/.ansible + ln -s $(pwd)/roles ~/.ansible/ + ln -s $(pwd)/plugins ~/.ansible/ + ls -l ~/.ansible/* + + - name: Show Vagrant version + run: | + vagrant --version + + - name: Run vagrant up + run: vagrant up + + - name: Get vagrant ssh config and IP addresses + run: | + vagrant ssh-config | tee "vagrant-ssh" | ./get_ip.sh > inventory/group_vars/all.yml + + - name: Ansible ping target hosts. + run: | + ansible -i inventory --ssh-extra-args "-F vagrant-ssh" -m ping all + + # Here is where you add tests... + - name: Test IPA server deploy + run: ansible-playbook -i inventory --ssh-extra-args "-F vagrant-ssh" playbooks/install-server.yml + + - name: Test IPA client deploy + run: ansible-playbook -i inventory --ssh-extra-args "-F vagrant-ssh" playbooks/install-client.yml + + #- name: Ensure server PTR records are available + # run: ansible-playbook -i inventory --ssh-extra-args "-F vagrant-ssh" ensure-reverse-dns.yaml + + - name: Test IPA replica deploy + run: ansible-playbook -i inventory --ssh-extra-args "-F vagrant-ssh" playbooks/install-replica.yml + + - name: Retrieve logs in case of ANY deploy failure + if: failure() + working-directory: . + run: | + ssh -F tests/multihost/vagrant-ssh server.ipa.test "sudo chmod a+r /var/log/*.log" + mkdir -p logs/server-logs + scp -F tests/multihost/vagrant-ssh vagrant@server.ipa.test:/var/log/{ipaserver,ipaclient}-install.log logs/server-logs || true + ssh -F tests/multihost/vagrant-ssh rep-01.ipa.test "sudo chmod a+r /var/log/*.log" + mkdir -p logs/replica-logs + scp -F tests/multihost/vagrant-ssh vagrant@rep-01.ipa.test:/var/log/{ipareplica,ipaclient}-install.log logs/replica-logs || true + ssh -F tests/multihost/vagrant-ssh cli-01.ipa.test "sudo chmod a+r /var/log/*.log" + mkdir -p logs/client-logs + scp -F tests/multihost/vagrant-ssh vagrant@cli-01.ipa.test:/var/log/ipaclient-install.log logs/client-logs || true + tar czvf multihost-logs.tar.gz logs + + - name: Save artifacts + if: failure() + # if: github.event.state == 'error' || github.event.state == 'failure' + uses: actions/upload-artifact@v3 + with: + path: multihost-logs.tar.gz + if-no-files-found: "ignore" + + # Cleanup + - name: Stop vagrant + run: vagrant destroy -f diff --git a/.gitignore b/.gitignore index 3e46ed6361..45a605204f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,8 @@ /.tox/ /.venv/ +# ignore Vagrant data +/.vagrant/ +/tests/multihost/vagrant-ssh + tests/logs/ diff --git a/tests/multihost/.gitignore b/tests/multihost/.gitignore new file mode 100644 index 0000000000..c8c1782b89 --- /dev/null +++ b/tests/multihost/.gitignore @@ -0,0 +1,2 @@ +/.vagrant/ + diff --git a/tests/multihost/README-vagrant.md b/tests/multihost/README-vagrant.md new file mode 100644 index 0000000000..f1235530d6 --- /dev/null +++ b/tests/multihost/README-vagrant.md @@ -0,0 +1,93 @@ +Multihost testing with Vagrant +============================== + +To test ipaserver role and ipabackup restore options, it is required that a target node without IPA installed is provided. To test ipareplica and ipaclient roles, it is required that a multihost environvent is available, and at least one target node does not have IPA installed. This environment must have proper networking configuration and some isolation for the tarkget nodes that is not provided by containers. + +By using Vagrant along with Github Workflows we can have nested virtualization, allowing the creation of virtual machine nodes that will play the roles of primary server, replicas and clients. The use of Vagrant also allows the use of a similar environment to run the tests in a developer's local machine, if desired. + +Github workflows only allows nested vintualization within _macOS_ runners \[[1]\]\[[2]\]. A nice side effect of using macOS runners is that there is some more available memory for the VMs \[[3]\], which might allow the use of a Windows node, or more replicas/clients in the future. + +The Ansible controller is the runner, a macOS host with the latest `ansible-core` version, installed through `pip`. Connection to the hosts is done through Vagrant `ssh-config` setup. + +To execute a playbook, use `ansible-playbook -i vagrant-inventory.yml --ssh-extra-args "-F vagrant-ssh" `. The current directory is `/tests/multihost`. + + +VM Configuration +---------------- + +Currently, only three VMs are used, and the hostnames and memory sizes cannot be changed. + +* Server: + * hostname: server.ipa.test + * RAM: 2500 MB +* Replica: + * hostname: rep-01.ipa.test + * private network ip: 192.168.56.102 + * RAM: 2500 MB +* Client: + * hostname: cli-01.ipa.test + * private network ip: 192.168.56.110 + * RAM: 768 MB + + +BASE Variables +---------------- + +| Name | Description | Type | Default +| `ipadm_password` | The password for the Directory Manager.| str | SomeDMpassword | +| `ipaadmin_password` | The password for the IPA admin user.| str | SomeADMINpassword | + + +Server Variables +---------------- + +| Name | Description | Type | Default +| `ipaserver_setup_kra`| Install and configure a KRA on this server. | bool | false | +| `ipaserver_setup_adtrust` | Configure AD Trust capability. | bool | false | +| `ipaserver_netbios_name` | The NetBIOS name for the IPA domain. | str | None | +| `ipaserver_setup_dns` | Configure an integrated DNS server, create DNS zone specified by domain. | bool | true | +| `ipaserver_auto_forwarders` | Add DNS forwarders configured in /etc/resolv.conf to the list of forwarders used by IPA DNS. | bool | true | +| `ipaserver_no_forwarders` | Do not add any DNS forwarders. Root DNS servers will be used instead. | bool | false | +| `ipaserver_forwarders` | Add DNS forwarders to the DNS configuration. | list of strings | \[\] | +| `ipaserver_auto_reverse` | Try to resolve reverse records and reverse zones for server IP addresses. | bool | true | +| `ipaserver_random_serial_numbers` | Enable use of random serial numbers for certificates. | bool | true | + +Also the following variables are always set: +```yaml +ipaserver_allow_zone_overlap: true +ipaserver_no_dnssec_validation: true +ipaserver_no_hbac_allow: true +``` + + +Replica Variables +---------------- + +| Name | Description | Type | Default +| `ipareplica_setup_kra`| Install and configure a KRA on this server. | bool | false | +| `ipareplica_setup_adtrust` | Configure AD Trust capability. | bool | false | +| `ipareplica_netbios_name` | The NetBIOS name for the IPA domain. | str | None | +| `ipareplica_setup_dns` | Configure an integrated DNS server, create DNS zone specified by domain. | bool | false | +| `ipareplica_auto_forwarders` | Add DNS forwarders configured in /etc/resolv.conf to the list of forwarders used by IPA DNS. | bool | true | +| `ipareplica_no_forwarders` | Do not add any DNS forwarders. Root DNS servers will be used instead. | bool | false | +| `ipareplica_forwarders` | Add DNS forwarders to the DNS configuration. | list of strings | \[\] | +| `ipareplica_auto_reverse` | Try to resolve reverse records and reverse zones for server IP addresses. | bool | true | +| `ipareplica_random_serial_numbers` | Enable use of random serial numbers for certificates. | bool | true | + + +Client Variables +---------------- + +Currently, no variables can be configured for the `ipaclient` role. + + +Caveats +------- + +As of this writing, there were some issues running Vagrant on `macos-latest`, and as it is transitioning from `macos-11` to `macos-12`, it was decided that the runner used will be pinned to `macos-12`. + + + +[1]: https://github.com/actions/runner-images/issues/183 +[2]: https://github.com/actions/runner-images/issues/433 +[3]: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources diff --git a/tests/multihost/Vagrantfile b/tests/multihost/Vagrantfile new file mode 100644 index 0000000000..6752d4b967 --- /dev/null +++ b/tests/multihost/Vagrantfile @@ -0,0 +1,55 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.box = "fedora/38-cloud-base" + + config.vm.provider :libvirt do |libvirt| + libvirt.qemu_use_session = false + libvirt.memory = 2500 + end + config.vm.provider :virtualbox do |virtualbox| + virtualbox.memory = 2500 + end + + # Prevent SharedFoldersEnableSymlinksCreate errors + config.vm.synced_folder ".", "/vagrant", disabled: true + # increase boot timeout (8 minutes). + config.vm.boot_timeout = 8 * 60 + + config.vm.define "server.ipa.test" do |server| + server.vm.hostname = "server.ipa.test" + server.vm.provision "shell", + inline: "hostnamectl set-hostname server.ipa.test" + server.vm.provision "shell", + inline: "echo $(hostname -I) server.ipa.test >> /etc/hosts" + server.vm.provision "shell", + inline: "dnf install --downloadonly -y freeipa-server python3-libselinux freeipa-server-dns freeipa-server-trust-ad firewalld" + end + + config.vm.define "rep-01.ipa.test" do |replica| + replica.vm.hostname="rep-01.ipa.test" + replica.vm.provision "shell", + inline: "hostnamectl set-hostname rep-01.ipa.test" + replica.vm.provision "shell", + inline: "echo $(hostname -I) rep-01.ipa.test >> /etc/hosts" + replica.vm.provision "shell", + inline: "dnf install --downloadonly -y freeipa-server python3-libselinux freeipa-server-dns freeipa-server-trust-ad firewalld" + end + + config.vm.define "cli-01.ipa.test" do |client| + client.vm.hostname="cli-01.ipa.test" + client.vm.provision "shell", + inline: "hostnamectl set-hostname cli-01.ipa.test" + client.vm.provision "shell", + inline: "dnf install --downloadonly -y freeipa-client python3-libselinux" + client.vm.provider :libvirt do |cmv| + cmv.memory = 768 + end + client.vm.provider :virtualbox do |cmv| + cmv.memory = 768 + end + end + +end + diff --git a/tests/multihost/ensure-reverse-dns.yaml b/tests/multihost/ensure-reverse-dns.yaml new file mode 100644 index 0000000000..651fb5176f --- /dev/null +++ b/tests/multihost/ensure-reverse-dns.yaml @@ -0,0 +1,18 @@ +--- +- name: Ensure IPA server has domain reverse zone and server PTR record. + hosts: ipaserver + become: no + gather_facts: no + + tasks: + - name: Ensure reverse zone is present. + ipadnszone: + ipaadmin_password: "{{ ipa_admin_password }}" + name_from_ip: "{{ server_ip }}" + + - name: Ensure server PTR record is set. + ipadnsrecord: + ipaadmin_password: "{{ ipa_admin_password }}" + zone_name: '{{ server_ip.split(".")[:-1][::-1] | join(".") }}.in-addr.arpa.' + name: '{{ server_ip.split(".")[-1] }}' + ptr_hostname: server.ipa.test. diff --git a/tests/multihost/get_ip.sh b/tests/multihost/get_ip.sh new file mode 100755 index 0000000000..460f6dd8c0 --- /dev/null +++ b/tests/multihost/get_ip.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +INPUT=${1:--} + +echo "---" +# shellcheck disable=SC2002 +cat "${INPUT}" | \ + grep HostName -B1 | \ + sed -e "/^--/d" \ + -e "/^Host/N;s/\n/:/;s/Host \([a-zA-Z0-9.]*\)/\1/;s/ *HostName \(.*\)/ \1/" \ + -e "s/server.*:/server_ip:/" \ + -e "s/cli-.*:/client_ip:/" \ + -e "s/rep-.*:/replica_ip:/" diff --git a/tests/multihost/inventory/group_vars/all.yml b/tests/multihost/inventory/group_vars/all.yml new file mode 100644 index 0000000000..9f52c4f584 --- /dev/null +++ b/tests/multihost/inventory/group_vars/all.yml @@ -0,0 +1,2 @@ +--- +# This file will be replaced during test execution. diff --git a/tests/multihost/inventory/vagrant-inventory.yml b/tests/multihost/inventory/vagrant-inventory.yml new file mode 100644 index 0000000000..a56ee7b666 --- /dev/null +++ b/tests/multihost/inventory/vagrant-inventory.yml @@ -0,0 +1,89 @@ +--- +all: + vars: + #ipa_dm_password: SomeDMpassword + #ipa_admin_password: SomeADMINpassword + # IPA variables + ipaserver_domain: ipa.test + ipaserver_realm: IPA.TEST + # ipareplica_realm: IPA.TEST + ipadm_password: "{{ lookup('env', 'ipadm_password') | default(SomeDMpassword, True) }}" + ipaadmin_password: "{{ lookup('env', 'ipaadmin_password') | default(SomeADMINpassword, True) }}" + children: + # define cluster + ipacluster: + children: + ipaserver: + ipareplicas: + ipaclients: + # IPA First (CA Renewal) Server + ipaserver: + hosts: + "server.ipa.test": + # Ansible connection configuration + ansible_ssh_user: vagrant + ansible_ssh_private_key_file: ".vagrant/machines/server/libvirt/private_key" + ansible_ssh_host_key_checking: false + # IPA Configuration. + vars: + # KRA + ipaserver_setup_kra: "{{ lookup('env', 'ipaserver_setup_kra') | default(false, True) | bool }}" + # DNS + ipaserver_setup_dns: "{{ lookup('env', 'ipaserver_setup_dns') | default(true, True) | bool }}" + ipaserver_auto_forwarders: "{{ lookup('env', 'ipaserver_auto_forwarders') | default(true, True) | bool }}" + ipaserver_no_forwarders: "{{ lookup('env', 'ipaserver_no_forwarders') | default(false, True) | bool }}" + ipaserver_forwarders: "{{ lookup('env', 'ipaserver_forwarders') | default([], True) }}" + ipaserver_auto_reverse: "{{ lookup('env', 'ipaserver_auto_reverse') | default(true, True) | bool }}" + # For easier setup of DNS keep it set to 'true' + ipaserver_allow_zone_overlap: true + # DNSSEC must be set to 'false' for AD trust + ipaserver_no_dnssec_validation: true + # trust vars + ipaserver_setup_adtrust: "{{ lookup('env', 'ipaserver_setup_adtrust') | default(false) | bool }}" + ipaserver_netbios_name: "{{ lookup('env', 'ipaserver_netbios_name') | default(omit) }}" + # disable 'allow all' HBAC rule + ipaserver_no_hbac_allow: true + # other vars + ipaserver_random_serial_numbers: "{{ lookup('env', 'ipaserver_random_serial_numbers:') | default(true, True) | bool }}" + # IPA Replica Servers + ipareplicas: + hosts: + "rep-01.ipa.test": + # Ansible connection configuration + ansible_ssh_user: vagrant + ansible_ssh_private_key_file: ".vagrant/machines/replica/libvirt/private_key" + ansible_ssh_host_key_checking: false + vars: + # CA backup + ipareplica_setup_ca: "{{ lookup('env', 'ipareplica_setup_ca') | default(false, True) | bool }}" + # KRA backup + ipareplica_setup_kra: "{{ lookup('env', 'ipareplica_setup_kra') | default(false, True) | bool }}" + # DNS backup + ipareplica_setup_dns: "{{ lookup('env', 'ipareplica_setup_dns') | default(false, True) | bool }}" + ipareplica_auto_forwarders: "{{ lookup('env', 'ipareplica_auto_forwarders') | default(true, True) | bool }}" + ipareplica_no_forwarders: "{{ lookup('env', 'ipareplica_no_forwarders') | default(false, True) | bool }}" + ipareplica_forwarders: "{{ lookup('env', 'ipareplica_forwarders') | default([], True) }}" + ipareplica_auto_reverse: "{{ lookup('env', 'ipareplica_auto_reverse') | default(true, True) | bool }}" + # Trust backup + ipareplica_setup_trust: "{{ lookup('env', 'ipaserver_setup_trust') | default(false) | bool }}" + ipareplica_netbios_name: "{{ lookup('env', 'ipaserver_netbios_name') | default(omit) }}" + # Update IP addressess + ipasssd_enable_dns_updates: true + # Automatically handle DNS nameservers (ansible-freeipa v1.9.0+) + ipaclient_configure_dns_resolver: "{{ ipaserver_setup_dns | default(false) }}" + ipaclient_dns_servers: ["{{ server_ip if (ipaserver_setup_dns | default(false)) else omit}}"] + # IPA Client hosts + ipaclients: + hosts: + "cli-01.ipa.test": + # Ansible connection configuration + ansible_ssh_user: vagrant + ansible_ssh_private_key_file: ".vagrant/machines/client/libvirt/private_key" + ansible_ssh_host_key_checking: false + # IPA Configuration. + vars: + # Add client DNS entries + ipasssd_enable_dns_updates: true + # Automatically handle DNS nameservers (ansible-freeipa v1.9.0+) + ipaclient_configure_dns_resolver: "{{ ipaserver_setup_dns | default(false) }}" + ipaclient_dns_servers: ["{{ server_ip if (ipaserver_setup_dns | default(false)) else omit}}"] diff --git a/tests/multihost/playbooks b/tests/multihost/playbooks new file mode 120000 index 0000000000..f32585fef8 --- /dev/null +++ b/tests/multihost/playbooks @@ -0,0 +1 @@ +../../playbooks \ No newline at end of file