From a94f9ce4fa32273fd0fad7492d36db4101423cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Oscarsson?= Date: Wed, 31 Oct 2018 07:06:54 +0100 Subject: [PATCH] Fixes #57 #58 #59 #60 #61 --- .gitignore | 5 +- CHANGELOG.md | 19 ++++-- tasks/configure-docker.yml | 11 +++- tasks/lvm-thinpool.yml | 6 ++ tests/Vagrantfile | 36 +++++++----- tests/prepare.yml | 24 ++++---- tests/prepare_storage.yml | 30 +++++----- tests/test.sh | 63 +++++++++++++------- tests/test_config_nightly.yml | 8 +++ tests/test_config_storage.yml | 2 +- tests/vagrant_config.yml | 106 +++++++++++++++++++--------------- 11 files changed, 194 insertions(+), 116 deletions(-) create mode 100644 tests/test_config_nightly.yml diff --git a/.gitignore b/.gitignore index ee5bc9e..ebbbd60 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ +/.project .vagrant -*.retry .vscode/ +*.retry *.vdi -/.project +*.log tests/yaml.sh tests/yamlparser.sh.inc tests/test-dbg.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index a0c0ac8..e26a601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased](../../releases/tag/X.Y.Z) +### Fixed + +- Docker startup fails in Fedora 28 because it cannot find pvcreate ([#58](../../issues/58)) +- LVM2 package is required to be installed when devicemapper is used ([#61](../../issues/61)) + +### Internal + +- Testing: Snapshotting used during testing by `test.sh` to speed up by avoiding unnecessary installs of Ansible and guest additions +- Add Fedora distributions to test suite ([#57](../../issues/57)) +- Fails to install VirtualBox guest additions on Fedora 29 beta ([#59](../../issues/59)) +- Replace currently used Vagrant boxes during testing with more official boxes ([#60](../../issues/60)) + ## [1.9.0](../../releases/tag/1.9.0) - 2018-10-24 ### Added @@ -25,11 +37,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support to install Docker Ansible module dependencies ([#48](../../issues/48)) - Add support to install packages after install via PiP or OS package manager ([#49](../../issues/49)) -### Changed +### Itnernal -- Testing: - - Improved structure in `vagrant_config.yml` for `test.sh` - - Improved `test.sh` with better limit functionality +- Testing: Improved structure in `vagrant_config.yml` for `test.sh` +- Testing: Improved `test.sh` with better limit functionality ### Fixed diff --git a/tasks/configure-docker.yml b/tasks/configure-docker.yml index 7eb5193..6c5b44d 100644 --- a/tasks/configure-docker.yml +++ b/tasks/configure-docker.yml @@ -46,7 +46,16 @@ append: true with_items: "{{ docker_users }}" -- name: Ensure thin-provisioning-tools is installed when devicemapper is used (Ubuntu) +- name: Ensure lvm2 is installed when devicemapper is used (Fedora/Debian) + become: true + package: + name: lvm2 + state: present + when: (_docker_os_dist == "Fedora" or _docker_os_dist == "Debian") and + docker_daemon_config['storage-driver'] is defined and + docker_daemon_config['storage-driver'] == 'devicemapper' + +- name: Ensure thin-provisioning-tools is installed when devicemapper is used (Ubuntu/Debian) become: true package: name: thin-provisioning-tools diff --git a/tasks/lvm-thinpool.yml b/tasks/lvm-thinpool.yml index 3bb50e0..c321bd1 100644 --- a/tasks/lvm-thinpool.yml +++ b/tasks/lvm-thinpool.yml @@ -1,3 +1,9 @@ +- name: Ensure lvm2 is installed + become: true + package: + name: lvm2 + state: present + - name: Create LVM volume group become: true lvg: diff --git a/tests/Vagrantfile b/tests/Vagrantfile index b8f7476..f9a0b91 100644 --- a/tests/Vagrantfile +++ b/tests/Vagrantfile @@ -11,6 +11,7 @@ unless File.exist?(testcase_file) exit 1 end vagrant_config = YAML.load_file("#{testcase_file}") +vagrant_devs = ["/dev/sda", "/dev/sdb", "/dev/sdc"] Vagrant.configure("2") do |config| config.vm.box = vagrant_config['box'] @@ -27,13 +28,8 @@ Vagrant.configure("2") do |config| unless File.exist?(docker_disk) vb.customize ['createhd', '--filename', docker_disk, '--size', 5 * 1024] end - vb.customize ['storageattach', :id, '--storagectl', vagrant_config['ide_ctl_name'], '--port', 1, '--device', 0, '--type', 'hdd', '--medium', docker_disk] + vb.customize ['storageattach', :id, '--storagectl', vagrant_config['storage_ctl'], '--port', vagrant_config['storage_port'], '--device', 0, '--type', 'hdd', '--medium', docker_disk] end - - # Print test ID - config.vm.provision "shell", - inline: "echo '--> Test: " + vagrant_config['id'] + "'" - # Prepare Ansible roles directory config.vm.provision "shell", inline: "test -d /etc/ansible/roles/haxorof.docker-ce || \ @@ -42,18 +38,30 @@ Vagrant.configure("2") do |config| # Prepare box config.vm.provision "ansible_local" do |ansible| - ansible.verbose = false - ansible.playbook = vagrant_config['prep_yml'] - ansible.limit = "test-host" + ansible.verbose = false + ansible.install_mode = "pip" + ansible.compatibility_mode = "2.0" + ansible.version = "latest" + ansible.playbook = vagrant_config['prep_yml'] + ansible.limit = "test-host" ansible.inventory_path = "hosts" - ansible.config_file = "/vagrant/ansible.cfg" + ansible.config_file = "/vagrant/ansible.cfg" + ansible.extra_vars = { + lvm_device: "#{vagrant_devs[vagrant_config['storage_port']]}" + } end # Test storage config config.vm.provision "ansible_local" do |ansible| - ansible.verbose = false - ansible.playbook = vagrant_config['test_yml'] - ansible.limit = "test-host" + ansible.verbose = false + ansible.install_mode = "pip" + ansible.compatibility_mode = "2.0" + ansible.version = "latest" + ansible.playbook = vagrant_config['test_yml'] + ansible.limit = "test-host" ansible.inventory_path = "hosts" - ansible.config_file = "/vagrant/ansible.cfg" + ansible.config_file = "/vagrant/ansible.cfg" + ansible.extra_vars = { + lvm_device: "#{vagrant_devs[vagrant_config['storage_port']]}" + } end end diff --git a/tests/prepare.yml b/tests/prepare.yml index 65206f4..ea6d3c5 100644 --- a/tests/prepare.yml +++ b/tests/prepare.yml @@ -1,14 +1,14 @@ --- - hosts: test-host - tasks: - - name: Ensure git is installed - package: - name: git - state: present - become: true - - name: Ensure Docker Bench Security is cloned - git: - repo: 'https://github.com/docker/docker-bench-security.git' - dest: /home/vagrant/docker-bench-security - version: master - when: docker_bench_security is defined + tasks: [] + # - name: Ensure git is installed + # package: + # name: git + # state: present + # become: true + # - name: Ensure Docker Bench Security is cloned + # git: + # repo: 'https://github.com/docker/docker-bench-security.git' + # dest: /home/vagrant/docker-bench-security + # version: master + # when: docker_bench_security is defined diff --git a/tests/prepare_storage.yml b/tests/prepare_storage.yml index b1a0c1a..fe67da1 100644 --- a/tests/prepare_storage.yml +++ b/tests/prepare_storage.yml @@ -1,42 +1,42 @@ --- - hosts: test-host tasks: - - name: Ensure git and parted is installed + - name: Ensure dependencies are installed package: name: "{{ item }}" state: present become: true with_items: - - git - parted + #- git - - name: Ensure Docker Bench Security is cloned - git: - repo: 'https://github.com/docker/docker-bench-security.git' - dest: /home/vagrant/docker-bench-security - version: master - when: docker_bench_security is defined + # - name: Ensure Docker Bench Security is cloned + # git: + # repo: 'https://github.com/docker/docker-bench-security.git' + # dest: /home/vagrant/docker-bench-security + # version: master + # when: docker_bench_security is defined - - name: Create partition sdb1 + - name: Create partition {{ lvm_device }}1 parted: - device: /dev/sdb + device: "{{ lvm_device }}" number: 1 flags: [ lvm ] part_end: 1GiB state: present become: true - - name: Create partition sdb2 + - name: Create partition {{ lvm_device }}2 parted: - device: /dev/sdb + device: "{{ lvm_device }}" number: 2 flags: [ lvm ] part_start: 1GiB state: present become: true - - name: Ensure file system is EXT4 for /dev/sdb1 + - name: Ensure file system is EXT4 for {{ lvm_device }}1 filesystem: fstype: ext4 - dev: /dev/sdb1 + dev: "{{ lvm_device }}1" become: true - name: Ensure /var/lib/docker directory exists file: @@ -46,7 +46,7 @@ - name: Ensure /var/lib/docker is mounted mount: path: /var/lib/docker - src: /dev/sdb1 + src: "{{ lvm_device }}1" fstype: ext4 state: mounted become: true diff --git a/tests/test.sh b/tests/test.sh index 15dd72d..083de42 100644 --- a/tests/test.sh +++ b/tests/test.sh @@ -61,13 +61,13 @@ VirtualBox() { exit 2 fi $VIRTUALBOX_CMD $@ 2>&1 - _exitCode=$? + local _exitCode=$? return $_exitCode } LogVirtualBoxVersion() { - _output=$(VirtualBox --version) - _exitCode=$? + local _output=$(VirtualBox --version) + local _exitCode=$? if [[ $_exitCode -eq 0 ]]; then Info "VirtualBox $_output" else @@ -82,13 +82,13 @@ Vagrant() { exit 2 fi $VAGRANT_CMD $@ 2>&1 - _exitCode=$? + local _exitCode=$? return $_exitCode } LogVagrantVersion() { - _output=$(Vagrant --version) - _exitCode=$? + local _output=$(Vagrant --version) + local _exitCode=$? if [[ $_exitCode -eq 0 ]]; then Info "$_output" else @@ -114,7 +114,7 @@ VagrantProvision() { VagrantDestroy() { Vagrant destroy -f - _exitCode=$? + local _exitCode=$? if [[ -f $VAGRANT_TESTCASE_FILE ]]; then rm $VAGRANT_TESTCASE_FILE fi @@ -122,8 +122,8 @@ VagrantDestroy() { } VagrantBoxAdd() { - cmdOutput=$(Vagrant box add --provider=virtualbox $1) - exitCode=$? + local cmdOutput=$(Vagrant box add --provider=virtualbox $1) + local exitCode=$? if [[ "$cmdOutput" == *force* ]]; then return 0 else @@ -135,11 +135,12 @@ VagrantBoxAdd() { } GenerateDoNothingConfig() { - _box_index=$1 - _test_index=$2 - cat << EOF > $VAGRANT_TESTCASE_FILE + local _box_index=$1 + local _test_index=$2 + cat << EOF > $VAGRANT_TESTCASE_FILE box: ${boxes__box[$_box_index]} -ide_ctl_name: ${boxes__ide_ctl_name[$_box_index]} +storage_ctl: ${boxes__storage_ctl[$_box_index]} +storage_port: ${boxes__storage_port[$_box_index]} vbguest_update: ${boxes__vbguest_update[$_box_index]} id: snapshot prep_yml: test_nothing.yml @@ -149,10 +150,21 @@ EOF VagrantSaveSnapshot() { GenerateDoNothingConfig $1 + local _exitCode=0 Vagrant up + let "_exitCode += $?" Vagrant halt - Vagrant snapshot save default base - return $? + let "_exitCode += $?" + Vagrant snapshot save default base -f + let "_exitCode += $?" + if [[ "$_exitCode" != "0" ]]; then + if [[ "$ON_FAILURE_KEEP" == "1" ]]; then + Info "VM is kept for debugging" + else + VagrantDestroy + fi + fi + return $_exitCode } VagrantRestoreSnapShot() { @@ -167,7 +179,7 @@ VagrantDeleteSnapshot() { DownloadBoxes() { Info "Downloading boxes..." - exitCode=0 + local exitCode=0 for box in ${boxes__box[*]}; do if [[ "$box" != *"$LIMIT_BOX"* ]]; then Skip "Download $box" @@ -185,11 +197,12 @@ DownloadBoxes() { } GenerateTestCaseConfig() { - _box_index=$1 - _test_index=$2 + local _box_index=$1 + local _test_index=$2 cat << EOF > $VAGRANT_TESTCASE_FILE box: ${boxes__box[$_box_index]} -ide_ctl_name: ${boxes__ide_ctl_name[$_box_index]} +storage_ctl: ${boxes__storage_ctl[$_box_index]} +storage_port: ${boxes__storage_port[$_box_index]} vbguest_update: ${boxes__vbguest_update[$_box_index]} id: ${tests__id[$_test_index]} prep_yml: ${tests__prep_yml[$_test_index]} @@ -199,16 +212,22 @@ EOF ExecuteTests() { Info "Starting tests..." - exitCode=0 + local exitCode=0 + local do_skip=0 for box_index in $(seq 0 `expr ${#boxes__box[@]} - 1`); do - box=${boxes__box[$box_index]} + local box=${boxes__box[$box_index]} if [[ "$box" != *"$LIMIT_BOX"* ]]; then Skip "(code:1) Test: Skipping box [$box]" continue fi VagrantSaveSnapshot $box_index + exitCode=$? + if [[ "$exitCode" != "0" ]]; then + Fail "Error when trying to create snapshot for box $box" + break + fi for index in $(seq 0 `expr ${#tests__name[@]} - 1`); do - test_name=${tests__name[$index]} + local test_name=${tests__name[$index]} if [[ "${tests__id[$index]}" != *"$LIMIT_TEST"* ]]; then Skip "(code:2) Test: $test_name [$box]" continue diff --git a/tests/test_config_nightly.yml b/tests/test_config_nightly.yml new file mode 100644 index 0000000..e577763 --- /dev/null +++ b/tests/test_config_nightly.yml @@ -0,0 +1,8 @@ +--- +- hosts: test-host + vars: + docker_channel: nightly + docker_users: + - vagrant + roles: + - haxorof.docker-ce diff --git a/tests/test_config_storage.yml b/tests/test_config_storage.yml index 8548bdc..36cd4fc 100644 --- a/tests/test_config_storage.yml +++ b/tests/test_config_storage.yml @@ -4,6 +4,6 @@ docker_daemon_config: storage-driver: devicemapper storage-opts: - - "dm.directlvm_device=/dev/sdb2" + - "dm.directlvm_device={{ lvm_device }}2" roles: - haxorof.docker-ce diff --git a/tests/vagrant_config.yml b/tests/vagrant_config.yml index 916a9ca..76b2c48 100644 --- a/tests/vagrant_config.yml +++ b/tests/vagrant_config.yml @@ -1,48 +1,64 @@ boxes: - - box: geerlingguy/centos7 - ide_ctl_name: IDE Controller - vbguest_update: false - - - box: geerlingguy/ubuntu1404 - ide_ctl_name: IDE Controller - vbguest_update: false - - - box: geerlingguy/ubuntu1604 - ide_ctl_name: IDE Controller - vbguest_update: false - - - box: geerlingguy/ubuntu1804 - ide_ctl_name: IDE Controller - vbguest_update: false - - - box: geerlingguy/debian8 - ide_ctl_name: IDE Controller - vbguest_update: false - - - box: geerlingguy/debian9 - ide_ctl_name: IDE Controller - vbguest_update: false - # - - # box: fedora/25-cloud-base - # ide_ctl_name: IDE - # vbguest_update: true - # - - # box: fedora/26-cloud-base - # ide_ctl_name: IDE - # vbguest_update: true - # - - # box: fedora/27-cloud-base - # ide_ctl_name: IDE - # vbguest_update: true - # - - # box: fedora/28-cloud-base - # ide_ctl_name: IDE - # vbguest_update: true - # - - # box: fedora/beta-29-cloud-base - # ide_ctl_name: IDE - # vbguest_update: true + box: centos/7 + storage_ctl: IDE + storage_port: 1 + vbguest_update: true + - + box: ubuntu/trusty64 + storage_ctl: SATAController + storage_port: 1 + vbguest_update: true + - + box: ubuntu/xenial64 + storage_ctl: SCSI + storage_port: 2 + vbguest_update: true + - + box: ubuntu/bionic64 + storage_ctl: SCSI + storage_port: 2 + vbguest_update: true + - + box: debian/wheezy64 + storage_ctl: SATA Controller + storage_port: 1 + vbguest_update: true + - + box: debian/jessie64 + storage_ctl: SATA Controller + storage_port: 1 + vbguest_update: true + - + box: debian/stretch64 + storage_ctl: SATA Controller + storage_port: 1 + vbguest_update: true + - + box: fedora/25-cloud-base + storage_ctl: IDE + storage_port: 1 + vbguest_update: true + - + box: fedora/26-cloud-base + storage_ctl: IDE + storage_port: 1 + vbguest_update: true + - + box: fedora/27-cloud-base + storage_ctl: IDE + storage_port: 1 + vbguest_update: true + - + box: fedora/28-cloud-base + storage_ctl: IDE + storage_port: 1 + vbguest_update: true + - + box: fedora/29-cloud-base + storage_ctl: IDE + storage_port: 1 + vbguest_update: true tests: - @@ -86,10 +102,10 @@ tests: name: Old version of Docker prep_yml: prepare.yml test_yml: test_old_version.yml - skip_boxes: ubuntu,debian + skip_boxes: ubuntu,debian,fedora - id: t_issue_42 name: Issue 42 prep_yml: prepare.yml test_yml: test_issue_42.yml - skip_boxes: centos + skip_boxes: centos,fedora