From d3954a559059bf17d049170270df5d71f8503d71 Mon Sep 17 00:00:00 2001
From: Etienne Champetier <champetier.etienne@gmail.com>
Date: Thu, 17 Dec 2020 11:07:09 -0500
Subject: [PATCH 1/8] [2.14] fix ci (#7021)

* fix flake8 errors in Kubespray CI - tox-inventory-builder

* fix flake8 errors in Kubespray CI - tox-inventory-builder

* Invalidate CRI-O kubic repo's cache

Signed-off-by: Victor Morales <v.morales@samsung.com>

* add support to configure pkg install retries

and use in CI job tf-ovh_ubuntu18-calico (due to it failing often)

* Switch Calico and Cilium image repos to Quay.io

Co-authored-by: Victor Morales <v.morales@samsung.com>
Co-authored-by: Barry Melbourne <9964974+bmelbourne@users.noreply.github.com>

Conflicts:
	roles/download/defaults/main.yml

* up vagrant box to fedora/33-cloud-base in cri-o molecule tests

(cherry picked from commit 06ec5393d770097c58a914ecf7a100251042bbd4)

* add Google proxy-mirror-cache for docker hub to CI tests

(cherry picked from commit d739a6bb2f935954510d6a83629584953db67fca)

* containerd docker hub registry mirror support

* containerd docker hub registry mirror support

* add docs

* fix typo

* fix yamllint

* fix indent in sample
and ansible-playbook param in testcases_run

* fix md

* mv common vars to tests/common/_docker_hub_registry_mirror.yml

* checkout vars to upgrade tests

(cherry picked from commit 4a8a52bad9ecd89d3879b37601112200a62bfcd9)

* Exclude .git/ from shellcheck

If a branch name contains '.sh', current shellcheck checks the branch
file under .git/ and outputs error because the format is not shell
script one.
This makes shellcheck exclude files under .git/ to avoid this issue.

(cherry picked from commit e2467d87b65ad9050bea022a2d514172520e48f3)

Co-authored-by: Hans Feldt <2808287+hafe@users.noreply.github.com>
Co-authored-by: Sergey <s.bondarev@southbridge.ru>
Co-authored-by: Kenichi Omichi <ken-oomichi@wx.jp.nec.com>
---
 .gitlab-ci.yml                                |  1 +
 .gitlab-ci/shellcheck.yml                     |  2 +-
 .../inventory_builder/tests/test_inventory.py | 22 ++++++-------
 docs/containerd.md                            | 31 +++++++++++++++++++
 .../sample/group_vars/all/containerd.yml      |  6 +++-
 .../containerd/templates/config.toml.j2       |  2 +-
 .../cri-o/molecule/default/molecule.yml       |  2 +-
 .../cri-o/tasks/crio_repo.yml                 |  1 +
 roles/download/defaults/main.yml              | 14 ++++-----
 roles/kubernetes/preinstall/defaults/main.yml |  3 ++
 .../preinstall/tasks/0070-system-packages.yml |  2 +-
 tests/common/_docker_hub_registry_mirror.yml  | 15 +++++++++
 tests/files/tf-ovh_ubuntu18-calico.yml        |  2 ++
 tests/scripts/testcases_run.sh                | 15 ++++-----
 14 files changed, 88 insertions(+), 30 deletions(-)
 create mode 100644 docs/containerd.md
 create mode 100644 tests/common/_docker_hub_registry_mirror.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5e34ef1d175..6d5c4bdce0d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,6 +15,7 @@ variables:
   MAGIC: "ci check this"
   TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
   CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
+  CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml"
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CONTAINER_ENGINE: docker
diff --git a/.gitlab-ci/shellcheck.yml b/.gitlab-ci/shellcheck.yml
index 96c576c342c..1b4e8fe2af7 100644
--- a/.gitlab-ci/shellcheck.yml
+++ b/.gitlab-ci/shellcheck.yml
@@ -12,5 +12,5 @@ shellcheck:
     - shellcheck --version
   script:
     # Run shellcheck for all *.sh except contrib/
-    - find . -name '*.sh' -not -path './contrib/*' | xargs shellcheck --severity error
+    - find . -name '*.sh' -not -path './contrib/*' -not -path './.git/*' | xargs shellcheck --severity error
   except: ['triggers', 'master']
diff --git a/contrib/inventory_builder/tests/test_inventory.py b/contrib/inventory_builder/tests/test_inventory.py
index aa03e7c6409..d76bb5474fc 100644
--- a/contrib/inventory_builder/tests/test_inventory.py
+++ b/contrib/inventory_builder/tests/test_inventory.py
@@ -51,7 +51,7 @@ def test_ensure_required_groups(self):
         groups = ['group1', 'group2']
         self.inv.ensure_required_groups(groups)
         for group in groups:
-            self.assertTrue(group in self.inv.yaml_config['all']['children'])
+            self.assertIn(group, self.inv.yaml_config['all']['children'])
 
     def test_get_host_id(self):
         hostnames = ['node99', 'no99de01', '01node01', 'node1.domain',
@@ -209,8 +209,8 @@ def test_purge_invalid_hosts(self):
             ('doesnotbelong2', {'whateveropts=ilike'})])
         self.inv.yaml_config['all']['hosts'] = existing_hosts
         self.inv.purge_invalid_hosts(proper_hostnames)
-        self.assertTrue(
-            bad_host not in self.inv.yaml_config['all']['hosts'].keys())
+        self.assertNotIn(
+            bad_host, self.inv.yaml_config['all']['hosts'].keys())
 
     def test_add_host_to_group(self):
         group = 'etcd'
@@ -227,8 +227,8 @@ def test_set_kube_master(self):
         host = 'node1'
 
         self.inv.set_kube_master([host])
-        self.assertTrue(
-            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+        self.assertIn(
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
 
     def test_set_all(self):
         hosts = OrderedDict([
@@ -246,8 +246,8 @@ def test_set_k8s_cluster(self):
 
         self.inv.set_k8s_cluster()
         for host in expected_hosts:
-            self.assertTrue(
-                host in
+            self.assertIn(
+                host,
                 self.inv.yaml_config['all']['children'][group]['children'])
 
     def test_set_kube_node(self):
@@ -255,16 +255,16 @@ def test_set_kube_node(self):
         host = 'node1'
 
         self.inv.set_kube_node([host])
-        self.assertTrue(
-            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+        self.assertIn(
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
 
     def test_set_etcd(self):
         group = 'etcd'
         host = 'node1'
 
         self.inv.set_etcd([host])
-        self.assertTrue(
-            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+        self.assertIn(
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
 
     def test_scale_scenario_one(self):
         num_nodes = 50
diff --git a/docs/containerd.md b/docs/containerd.md
new file mode 100644
index 00000000000..58fd44d8f68
--- /dev/null
+++ b/docs/containerd.md
@@ -0,0 +1,31 @@
+# conrainerd
+
+[containerd] An industry-standard container runtime with an emphasis on simplicity, robustness and portability
+Kubespray supports basic functionality for using containerd as the default container runtime in a cluster.
+
+_To use the containerd container runtime set the following variables:_
+
+## k8s-cluster.yml
+
+```yaml
+container_manager: containerd
+```
+
+## Containerd config
+
+Example: define registry mirror for docker hub
+
+```yaml
+containerd_config:
+  grpc:
+    max_recv_message_size: 16777216
+    max_send_message_size: 16777216
+  debug:
+    level: ""
+  registries:
+    "docker.io":
+      - "https://mirror.gcr.io"
+      - "https://registry-1.docker.io"
+```
+
+[containerd]: https://containerd.io/
diff --git a/inventory/sample/group_vars/all/containerd.yml b/inventory/sample/group_vars/all/containerd.yml
index 2fc66b636d4..0f1e97749e1 100644
--- a/inventory/sample/group_vars/all/containerd.yml
+++ b/inventory/sample/group_vars/all/containerd.yml
@@ -1,6 +1,8 @@
 ---
 # Please see roles/container-engine/containerd/defaults/main.yml for more configuration options
 
+# Example: define registry mirror for docker hub
+
 # containerd_config:
 #   grpc:
 #     max_recv_message_size: 16777216
@@ -8,7 +10,9 @@
 #   debug:
 #     level: ""
 #   registries:
-#     "docker.io": "https://registry-1.docker.io"
+#     "docker.io":
+#       - "https://mirror.gcr.io"
+#       - "https://registry-1.docker.io"
 #   max_container_log_line_size: -1
 #   metrics:
 #     address: ""
diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2
index ceccaa2fc4a..671af29e48d 100644
--- a/roles/container-engine/containerd/templates/config.toml.j2
+++ b/roles/container-engine/containerd/templates/config.toml.j2
@@ -62,7 +62,7 @@ disabled_plugins = ["restart"]
 [plugins.cri.registry.mirrors]
 {% for registry, addr in containerd_config.registries.items() %}
 [plugins.cri.registry.mirrors."{{ registry }}"]
-  endpoint = ["{{ addr }}"]
+  endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
 {% endfor %}
 {% endif %}
 
diff --git a/roles/container-engine/cri-o/molecule/default/molecule.yml b/roles/container-engine/cri-o/molecule/default/molecule.yml
index e2132134d0e..a6c36acbac7 100644
--- a/roles/container-engine/cri-o/molecule/default/molecule.yml
+++ b/roles/container-engine/cri-o/molecule/default/molecule.yml
@@ -26,7 +26,7 @@ platforms:
     groups:
       - kube-master
   - name: fedora
-    box: fedora/31-cloud-base
+    box: fedora/33-cloud-base
     cpus: 2
     memory: 1024
     groups:
diff --git a/roles/container-engine/cri-o/tasks/crio_repo.yml b/roles/container-engine/cri-o/tasks/crio_repo.yml
index b5cbd5b88aa..33efc317e78 100644
--- a/roles/container-engine/cri-o/tasks/crio_repo.yml
+++ b/roles/container-engine/cri-o/tasks/crio_repo.yml
@@ -29,6 +29,7 @@
     baseurl: http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/
     gpgcheck: yes
     gpgkey: http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/repodata/repomd.xml.key
+    keepcache: false
   when: ansible_distribution in ["CentOS"]
 
 - name: Add CRI-O kubic repo
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index d0d60b6fb18..3ec6606ee75 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -419,13 +419,13 @@ etcd_image_repo: "{{ quay_image_repo }}/coreos/etcd"
 etcd_image_tag: "{{ etcd_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 flannel_image_repo: "{{ quay_image_repo }}/coreos/flannel"
 flannel_image_tag: "{{ flannel_version }}"
-calico_node_image_repo: "{{ docker_image_repo }}/calico/node"
+calico_node_image_repo: "{{ quay_image_repo }}/calico/node"
 calico_node_image_tag: "{{ calico_version }}"
-calico_cni_image_repo: "{{ docker_image_repo }}/calico/cni"
+calico_cni_image_repo: "{{ quay_image_repo }}/calico/cni"
 calico_cni_image_tag: "{{ calico_cni_version }}"
-calico_policy_image_repo: "{{ docker_image_repo }}/calico/kube-controllers"
+calico_policy_image_repo: "{{ quay_image_repo }}/calico/kube-controllers"
 calico_policy_image_tag: "{{ calico_policy_version }}"
-calico_typha_image_repo: "{{ docker_image_repo }}/calico/typha"
+calico_typha_image_repo: "{{ quay_image_repo }}/calico/typha"
 calico_typha_image_tag: "{{ calico_typha_version }}"
 pod_infra_image_repo: "{{ kube_image_repo }}/pause"
 pod_infra_image_tag: "{{ pod_infra_version }}"
@@ -450,11 +450,11 @@ contiv_etcd_init_image_repo: "{{ docker_image_repo }}/ferest/etcd-initer"
 contiv_etcd_init_image_tag: latest
 contiv_ovs_image_repo: "{{ docker_image_repo }}/contiv/ovs"
 contiv_ovs_image_tag: "latest"
-cilium_image_repo: "{{ docker_image_repo }}/cilium/cilium"
+cilium_image_repo: "{{ quay_image_repo }}/cilium/cilium"
 cilium_image_tag: "{{ cilium_version }}"
-cilium_init_image_repo: "{{ docker_image_repo }}/cilium/cilium-init"
+cilium_init_image_repo: "{{ quay_image_repo }}/cilium/cilium-init"
 cilium_init_image_tag: "2019-04-05"
-cilium_operator_image_repo: "{{ docker_image_repo }}/cilium/operator"
+cilium_operator_image_repo: "{{ quay_image_repo }}/cilium/operator"
 cilium_operator_image_tag: "{{ cilium_version }}"
 kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn"
 kube_ovn_container_image_tag: "{{ kube_ovn_version }}"
diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml
index f0b54c444a4..5bf702dba25 100644
--- a/roles/kubernetes/preinstall/defaults/main.yml
+++ b/roles/kubernetes/preinstall/defaults/main.yml
@@ -53,3 +53,6 @@ minimal_node_memory_mb: 1024
 minimal_master_memory_mb: 1500
 
 yum_repo_dir: /etc/yum.repos.d
+
+# number of times package install task should be retried
+pkg_install_retries: 4
diff --git a/roles/kubernetes/preinstall/tasks/0070-system-packages.yml b/roles/kubernetes/preinstall/tasks/0070-system-packages.yml
index fa5f5f0f3fa..0b78603c310 100644
--- a/roles/kubernetes/preinstall/tasks/0070-system-packages.yml
+++ b/roles/kubernetes/preinstall/tasks/0070-system-packages.yml
@@ -77,7 +77,7 @@
     state: latest
   register: pkgs_task_result
   until: pkgs_task_result is succeeded
-  retries: 4
+  retries: "{{ pkg_install_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   when: not (ansible_os_family in ["Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos)
   tags:
diff --git a/tests/common/_docker_hub_registry_mirror.yml b/tests/common/_docker_hub_registry_mirror.yml
new file mode 100644
index 00000000000..3dadb08ff67
--- /dev/null
+++ b/tests/common/_docker_hub_registry_mirror.yml
@@ -0,0 +1,15 @@
+---
+docker_registry_mirrors:
+  - "https://mirror.gcr.io"
+
+containerd_config:
+  grpc:
+    max_recv_message_size: 16777216
+    max_send_message_size: 16777216
+  debug:
+    level: ""
+  registries:
+    "docker.io":
+      - "https://mirror.gcr.io"
+      - "https://registry-1.docker.io"
+  max_container_log_line_size: -1
diff --git a/tests/files/tf-ovh_ubuntu18-calico.yml b/tests/files/tf-ovh_ubuntu18-calico.yml
index 43ef55aa34c..704e21735bd 100644
--- a/tests/files/tf-ovh_ubuntu18-calico.yml
+++ b/tests/files/tf-ovh_ubuntu18-calico.yml
@@ -2,6 +2,8 @@
 dns_min_replicas: 1
 deploy_netchecker: true
 sonobuoy_enabled: true
+pkg_install_retries: 10
+retry_stagger: 10
 
 # Ignore ping errors
 ignore_assert_errors: true
diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh
index 658ca389040..9f9870b5767 100755
--- a/tests/scripts/testcases_run.sh
+++ b/tests/scripts/testcases_run.sh
@@ -42,6 +42,7 @@ fi
 test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout "$KUBESPRAY_VERSION"
 # Checkout the CI vars file so it is available
 test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
+test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" ${CI_TEST_REGISTRY_MIRROR}
 
 # Install mitogen ansible plugin
 if [ "${MITOGEN_ENABLE}" = "true" ]; then
@@ -51,20 +52,20 @@ if [ "${MITOGEN_ENABLE}" = "true" ]; then
 fi
 
 # Create cluster
-ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 
 # Repeat deployment if testing upgrade
 if [ "${UPGRADE_TEST}" != "false" ]; then
   test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml"
   test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml"
   git checkout "${CI_BUILD_REF}"
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK
 fi
 
 # Test control plane recovery
 if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml
 fi
 
 # Tests Cases
@@ -88,7 +89,7 @@ ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} --limit "all:!fake_
 
 ## Idempotency checks 1/5 (repeat deployment)
 if [ "${IDEMPOT_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 fi
 
 ## Idempotency checks 2/5 (Advanced DNS checks)
@@ -98,12 +99,12 @@ fi
 
 ## Idempotency checks 3/5 (reset deployment)
 if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
 fi
 
 ## Idempotency checks 4/5 (redeploy after reset)
 if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 fi
 
 ## Idempotency checks 5/5 (Advanced DNS checks)

From af84e5609971b4f4f4b5e61615e53046e1692c7d Mon Sep 17 00:00:00 2001
From: Etienne Champetier <champetier.etienne@gmail.com>
Date: Fri, 18 Dec 2020 09:00:25 -0500
Subject: [PATCH 2/8] Fix nf_conntrack_ipv4 modprobe (#7014)

RedHat 8.3 merged nf_conntrack_ipv4 in nf_conntrack but still advertise 4.18
so just try to modprobe and decide depending on the success
Also nf_conntrack is a dependency of ip_vs, so no need to care about it

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>
(cherry picked from commit 00e0f3bd2bf3ce4f83359c30e34545f32af20d20)
---
 roles/kubernetes/node/tasks/main.yml | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml
index 8c1659f7670..32623c91ff4 100644
--- a/roles/kubernetes/node/tasks/main.yml
+++ b/roles/kubernetes/node/tasks/main.yml
@@ -103,23 +103,13 @@
   tags:
     - kube-proxy
 
-- name: Modprobe nf_conntrack_ipv4 for kernels < 4.19
+- name: Modprobe nf_conntrack_ipv4
   modprobe:
     name: nf_conntrack_ipv4
     state: present
-  register: enable_nf_conntrack
+  register: modprobe_nf_conntrack_ipv4
+  ignore_errors: yes
   when:
-    - ansible_kernel.split('.')[0:3] | join('.')  < '4.19'
-    - kube_proxy_mode == 'ipvs'
-  tags:
-    - kube-proxy
-
-- name: Modprobe nf_conntrack for kernels >= 4.19
-  modprobe:
-    name: nf_conntrack
-    state: present
-  when:
-    - ansible_kernel.split('.')[0:3] | join('.')  >= '4.19'
     - kube_proxy_mode == 'ipvs'
   tags:
     - kube-proxy
@@ -132,9 +122,7 @@
       ip_vs_rr
       ip_vs_wrr
       ip_vs_sh
-      {% if enable_nf_conntrack is failed -%}
-      nf_conntrack
-      {%-   else -%}
+      {% if modprobe_nf_conntrack_ipv4 is success -%}
       nf_conntrack_ipv4
       {%-   endif -%}
   when: kube_proxy_mode == 'ipvs'

From 6d37c3cde67cd25b4dc34a907728e849e3b323e2 Mon Sep 17 00:00:00 2001
From: Etienne Champetier <champetier.etienne@gmail.com>
Date: Tue, 22 Dec 2020 07:48:28 -0500
Subject: [PATCH 3/8] Bump nodelocaldns to 1.16.0 (#7068)

This new version uses the same base image as kube-proxy
(k8s.gcr.io/build-image/debian-iptables)
This allow to automatically pick iptables-legacy or iptables-nft,
and be compatible with RHEL/CentOS 8
https://github.com/kubernetes/dns/pull/367

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>
(cherry picked from commit e909f84966936ccb91b714b263aaf9af654b7bc0)
---
 docs/centos8.md                  | 2 +-
 roles/download/defaults/main.yml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/centos8.md b/docs/centos8.md
index d22b8184ee0..411c326ab38 100644
--- a/docs/centos8.md
+++ b/docs/centos8.md
@@ -2,7 +2,7 @@
 
 RHEL / CentOS 8 ships only with iptables-nft (ie without iptables-legacy)
 The only tested configuration for now is using Calico CNI
-You need to use K8S 1.17+ and to add `calico_iptables_backend: "NFT"` to your configuration
+You need to use K8S 1.17+ and to add `calico_iptables_backend: "NFT"` or `calico_iptables_backend: "Auto"` to your configuration
 
 If you have containers that are using iptables in the host network namespace (`hostNetwork=true`),
 you need to ensure they are using iptables-nft.
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index 3ec6606ee75..b20399150b6 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -478,8 +478,8 @@ coredns_version: "1.6.7"
 coredns_image_repo: "{{ docker_image_repo }}/coredns/coredns"
 coredns_image_tag: "{{ coredns_version }}"
 
-nodelocaldns_version: "1.15.13"
-nodelocaldns_image_repo: "{{ kube_image_repo }}/k8s-dns-node-cache"
+nodelocaldns_version: "1.16.0"
+nodelocaldns_image_repo: "{{ kube_image_repo }}/dns/k8s-dns-node-cache"
 nodelocaldns_image_tag: "{{ nodelocaldns_version }}"
 
 dnsautoscaler_version: 1.8.1

From c267d427ce122316ab9ec334b1a558d5486da6fb Mon Sep 17 00:00:00 2001
From: Etienne Champetier <champetier.etienne@gmail.com>
Date: Tue, 22 Dec 2020 07:54:26 -0500
Subject: [PATCH 4/8] Fix proxy and module_hotfixes (#7067)

This fixes the Containerd + EL8 case that was missed in 7d1ab3374e0b8c35ecbd7fb2339fc93c367c5c19

On CentOS 8 with proxy ansible render inline `proxy` and `module_hotfixes` options.

For example:
```
proxy=http://127.0.0.1:3128module_hotfixes=True
```

But expected result:
```
proxy=http://127.0.0.1:3128
module_hotfixes=True
```

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>
(cherry picked from commit 03f316e7a242d75db35e7e9f72f3f08a28b188f3)
---
 .../containerd/templates/rh_containerd.repo.j2           | 8 ++++++--
 .../container-engine/docker/templates/rh_docker.repo.j2  | 9 ++++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/roles/container-engine/containerd/templates/rh_containerd.repo.j2 b/roles/container-engine/containerd/templates/rh_containerd.repo.j2
index 81f9c37d5a6..a8a04f60f78 100644
--- a/roles/container-engine/containerd/templates/rh_containerd.repo.j2
+++ b/roles/container-engine/containerd/templates/rh_containerd.repo.j2
@@ -5,5 +5,9 @@ enabled=1
 gpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }}
 keepcache={{ docker_rpm_keepcache | default('1') }}
 gpgkey={{ docker_rh_repo_gpgkey }}
-{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}
-{% if ansible_os_family == "RedHat" and ansible_distribution_major_version|int == 8 %}module_hotfixes=True{% endif %}
+{% if http_proxy is defined %}
+proxy={{ http_proxy }}
+{% endif %}
+{% if ansible_os_family == "RedHat" and ansible_distribution_major_version|int == 8 %}
+module_hotfixes=True
+{% endif %}
diff --git a/roles/container-engine/docker/templates/rh_docker.repo.j2 b/roles/container-engine/docker/templates/rh_docker.repo.j2
index fe01ed2e6d4..a8a04f60f78 100644
--- a/roles/container-engine/docker/templates/rh_docker.repo.j2
+++ b/roles/container-engine/docker/templates/rh_docker.repo.j2
@@ -5,6 +5,9 @@ enabled=1
 gpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }}
 keepcache={{ docker_rpm_keepcache | default('1') }}
 gpgkey={{ docker_rh_repo_gpgkey }}
-{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}
-
-{% if ansible_os_family == "RedHat" and ansible_distribution_major_version|int == 8 %}module_hotfixes=True{% endif %}
+{% if http_proxy is defined %}
+proxy={{ http_proxy }}
+{% endif %}
+{% if ansible_os_family == "RedHat" and ansible_distribution_major_version|int == 8 %}
+module_hotfixes=True
+{% endif %}

From 11b72e2408f875be182e4c8ce1fe5442281d9223 Mon Sep 17 00:00:00 2001
From: Rick Haan <RickHaan@users.noreply.github.com>
Date: Tue, 26 Jan 2021 16:18:34 +0100
Subject: [PATCH 5/8] [2.14] Backport: 6758, 6853 and 7003 to fix CRI-O pkg
 (#7209)

* cherry-pick bump crio version to 1.19 (#6758)

cherry-pick modifications:
* keep default to 1.17 as release 2.14 came with
* don't change readme with newer versions

* bump crio version to 1.19

* crio package name has changed for debian/ubuntu
* crio upgrade does not work, see #6757

* update crio info in docs

* Install cri-o with package version (#6853)

and thereby support upgrade from e.g. 1.18.x to 1.19.y

Included OSes:
- Centos7/8
- Ubuntu18/20

New variables for overriding by default installed packages:
- centos_crio_packages
- ubuntu_crio_packages

* add support crio version for varios k8s vers (#7003)

* add support crio version for various k8s vers

* regexp in pkg versions

Co-authored-by: Hans Feldt <2808287+hafe@users.noreply.github.com>
Co-authored-by: Sergey <s.bondarev@southbridge.ru>
---
 docs/cri-o.md                                   |  3 +--
 roles/container-engine/cri-o/defaults/main.yml  |  1 +
 .../container-engine/cri-o/tasks/crio_repo.yml  | 17 ++++++++++++-----
 roles/container-engine/cri-o/vars/centos-7.yml  | 13 ++++++++++---
 roles/container-engine/cri-o/vars/centos-8.yml  | 12 ++++++++++--
 roles/container-engine/cri-o/vars/debian.yml    | 16 +++++++++++++---
 roles/container-engine/cri-o/vars/fedora.yml    |  7 +++++++
 roles/container-engine/cri-o/vars/ubuntu.yml    | 15 +++++++++++++--
 8 files changed, 67 insertions(+), 17 deletions(-)

diff --git a/docs/cri-o.md b/docs/cri-o.md
index b9ffb2a46ec..4fbb1ecb698 100644
--- a/docs/cri-o.md
+++ b/docs/cri-o.md
@@ -4,10 +4,9 @@
 Kubespray supports basic functionality for using CRI-O as the default container runtime in a cluster.
 
 * Kubernetes supports CRI-O on v1.11.1 or later.
-* Helm and other tools may not function as normal due to dependency on Docker.
 * `scale.yml` and `upgrade-cluster.yml` are not supported on clusters using CRI-O.
 
-_To use CRI-O instead of Docker, set the following variables:_
+_To use the CRI-O container runtime set the following variables:_
 
 ## all.yml
 
diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml
index 84b7da88aaf..40b666f1f33 100644
--- a/roles/container-engine/cri-o/defaults/main.yml
+++ b/roles/container-engine/cri-o/defaults/main.yml
@@ -21,6 +21,7 @@ crio_stream_port: "10010"
 crio_required_version: "{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
 
 crio_kubernetes_version_matrix:
+  "1.19": "1.19"
   "1.18": "1.18"
   "1.17": "1.17"
   "1.16": "1.16"
diff --git a/roles/container-engine/cri-o/tasks/crio_repo.yml b/roles/container-engine/cri-o/tasks/crio_repo.yml
index 33efc317e78..4993bedaa5e 100644
--- a/roles/container-engine/cri-o/tasks/crio_repo.yml
+++ b/roles/container-engine/cri-o/tasks/crio_repo.yml
@@ -5,7 +5,7 @@
     crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
   when: ansible_os_family == "Debian"
 
-- name: Add CRI-O kubic repo key
+- name: Add CRI-O kubic apt repo key
   apt_key:
     url: "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/{{ crio_kubic_debian_repo_name }}/Release.key"
     state: present
@@ -15,14 +15,21 @@
   retries: 4
   delay: "{{ retry_stagger | d(3) }}"
 
-- name: Add CRI-O kubic repo
+- name: Add CRI-O kubic apt repo
   apt_repository:
     repo: "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ crio_kubic_debian_repo_name }}/ /"
     state: present
-    filename: devel:kubic:libcontainers:stable
+    filename: devel-kubic-libcontainers-stable
   when: crio_kubic_debian_repo_name is defined
 
-- name: Add CRI-O kubic repo
+- name: Add CRI-O kubic cri-o apt repo
+  apt_repository:
+    repo: "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: present
+    filename: devel-kubic-libcontainers-stable-cri-o
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Add CRI-O kubic yum repo
   yum_repository:
     name: devel_kubic_libcontainers_stable
     description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever)
@@ -32,7 +39,7 @@
     keepcache: false
   when: ansible_distribution in ["CentOS"]
 
-- name: Add CRI-O kubic repo
+- name: Add CRI-O kubic yum repo
   yum_repository:
     name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
     description: "CRI-O {{ crio_version }} (CentOS_$releasever)"
diff --git a/roles/container-engine/cri-o/vars/centos-7.yml b/roles/container-engine/cri-o/vars/centos-7.yml
index 740adbc6b11..f2548560d6b 100644
--- a/roles/container-engine/cri-o/vars/centos-7.yml
+++ b/roles/container-engine/cri-o/vars/centos-7.yml
@@ -1,5 +1,12 @@
 ---
+crio_versioned_pkg:
+  "1.19":
+    - "cri-o-1.19.*"
+  "1.18":
+    - "cri-o-1.18.*"
+  "1.17":
+    - "cri-o-1.17.*"
 
-crio_packages:
-  - cri-o
-  - oci-systemd-hook
+default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
+
+crio_packages: "{{ centos_crio_packages | default(default_crio_packages) }}"
diff --git a/roles/container-engine/cri-o/vars/centos-8.yml b/roles/container-engine/cri-o/vars/centos-8.yml
index 1211808799c..f2548560d6b 100644
--- a/roles/container-engine/cri-o/vars/centos-8.yml
+++ b/roles/container-engine/cri-o/vars/centos-8.yml
@@ -1,4 +1,12 @@
 ---
+crio_versioned_pkg:
+  "1.19":
+    - "cri-o-1.19.*"
+  "1.18":
+    - "cri-o-1.18.*"
+  "1.17":
+    - "cri-o-1.17.*"
 
-crio_packages:
-  - cri-o
+default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
+
+crio_packages: "{{ centos_crio_packages | default(default_crio_packages) }}"
diff --git a/roles/container-engine/cri-o/vars/debian.yml b/roles/container-engine/cri-o/vars/debian.yml
index 62c966a1138..792c21ff049 100644
--- a/roles/container-engine/cri-o/vars/debian.yml
+++ b/roles/container-engine/cri-o/vars/debian.yml
@@ -1,7 +1,17 @@
 ---
+# Debian-10 has pkg only for cri-o 1.19
+crio_kubernetes_version_matrix:
+  "1.19": "1.19"
+  "1.18": "1.19"
+  "1.17": "1.19"
 
-crio_packages:
-  - "cri-o-{{ crio_version }}"
-  - runc
+crio_versioned_pkg:
+  "1.19":
+    - "cri-o=1.19*"
+    - cri-o-runc
+
+default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
+
+crio_packages: "{{ debian_crio_packages | default(default_crio_packages) }}"
 
 crio_runc_path: /usr/sbin/runc
diff --git a/roles/container-engine/cri-o/vars/fedora.yml b/roles/container-engine/cri-o/vars/fedora.yml
index e8efe8ac86d..fab1a5e9cbd 100644
--- a/roles/container-engine/cri-o/vars/fedora.yml
+++ b/roles/container-engine/cri-o/vars/fedora.yml
@@ -4,3 +4,10 @@ crio_packages:
   - cri-tools
 
 crio_conmon: /usr/libexec/crio/conmon
+
+# TODO: remove crio_kubernetes_version_matrix and crio_version once Fedora supports 1.19
+crio_kubernetes_version_matrix:
+  "1.18": "1.18"
+  "1.17": "1.17"
+
+crio_version: "{{ crio_kubernetes_version_matrix[crio_required_version] | default('1.17') }}"
diff --git a/roles/container-engine/cri-o/vars/ubuntu.yml b/roles/container-engine/cri-o/vars/ubuntu.yml
index 5495f193a6b..b26df5a0b4e 100644
--- a/roles/container-engine/cri-o/vars/ubuntu.yml
+++ b/roles/container-engine/cri-o/vars/ubuntu.yml
@@ -1,7 +1,18 @@
 ---
+crio_versioned_pkg:
+  "1.19":
+    - "cri-o=1.19*"
+    - cri-o-runc
+  "1.18":
+    - "cri-o=1.18*"
+    - cri-o-runc
+  "1.17":
+    - "cri-o=1.17*"
+    - cri-o-runc
 
-crio_packages:
-  - "cri-o-{{ crio_version }}"
+default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
+
+crio_packages: "{{ ubuntu_crio_packages | default(default_crio_packages) }}"
 
 crio_runc_path: /usr/sbin/runc
 

From 7e419310ce38a5eeaed5a7db4f497c484a32c9d6 Mon Sep 17 00:00:00 2001
From: Ryler Hockenbury <rhockenbury@users.noreply.github.com>
Date: Wed, 27 Jan 2021 06:47:40 -0500
Subject: [PATCH 6/8] Update azure cloud config (#7208) (#7220)

* Allow configureable vni and port for flannel overlay

* additional options for azure cloud config
---
 docs/azure.md                                    | 16 ++++++++++++++++
 inventory/sample/group_vars/all/azure.yml        |  2 ++
 .../cloud-configs/azure-cloud-config.j2          |  8 +++++---
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/docs/azure.md b/docs/azure.md
index d1baccc1823..8bd4c898ccf 100644
--- a/docs/azure.md
+++ b/docs/azure.md
@@ -42,6 +42,10 @@ The type of the vm. Supported values are `standard` or `vmss`. If vm is type of
 
 The name of the virtual network your instances are in, can be retrieved via `az network vnet list`
 
+### azure\_vnet\_resource\_group
+
+The name of the resource group that contains the vnet.
+
 ### azure\_subnet\_name
 
 The name of the subnet your instances are in, can be retrieved via `az network vnet subnet list --resource-group RESOURCE_GROUP --vnet-name VNET_NAME`
@@ -50,6 +54,18 @@ The name of the subnet your instances are in, can be retrieved via `az network v
 
 The name of the network security group your instances are in, can be retrieved via `az network nsg list`
 
+### azure\_security\_group\_resource\_group
+
+The name of the resource group that contains the network security group.  Defaults to `azure_vnet_resource_group`
+
+### azure\_route\_table\_name
+
+The name of the route table used with your instances.
+
+### azure\_route\_table\_resource\_group
+
+The name of the resource group that contains the route table.  Defaults to `azure_vnet_resource_group`
+
 ### azure\_aad\_client\_id + azure\_aad\_client\_secret
 
 These will have to be generated first:
diff --git a/inventory/sample/group_vars/all/azure.yml b/inventory/sample/group_vars/all/azure.yml
index b9daeb91a79..46517d3488a 100644
--- a/inventory/sample/group_vars/all/azure.yml
+++ b/inventory/sample/group_vars/all/azure.yml
@@ -10,9 +10,11 @@
 # azure_location:
 # azure_subnet_name:
 # azure_security_group_name:
+# azure_security_group_resource_group:
 # azure_vnet_name:
 # azure_vnet_resource_group:
 # azure_route_table_name:
+# azure_route_table_resource_group:
 # supported values are 'standard' or 'vmss'
 # azure_vmtype: standard
 
diff --git a/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2 b/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2
index c3964da1758..2b1c101aa5d 100644
--- a/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2
+++ b/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2
@@ -8,17 +8,19 @@
   "location": "{{ azure_location }}",
   "subnetName": "{{ azure_subnet_name }}",
   "securityGroupName": "{{ azure_security_group_name }}",
+  "securityGroupResourceGroup": "{{ azure_security_group_resource_group | default(azure_vnet_resource_group) }}",
   "vnetName": "{{ azure_vnet_name }}",
   "vnetResourceGroup": "{{ azure_vnet_resource_group }}",
   "routeTableName": "{{ azure_route_table_name }}",
+  "routeTableResourceGroup": "{{ azure_route_table_resource_group | default(azure_vnet_resource_group) }}",
   "vmType": "{{ azure_vmtype }}",
 {% if azure_primary_availability_set_name is defined %}
   "primaryAvailabilitySetName": "{{ azure_primary_availability_set_name }}",
 {%endif%}
-  "useInstanceMetadata": {{azure_use_instance_metadata }},
+  "useInstanceMetadata": {{azure_use_instance_metadata | lower }},
 {% if azure_loadbalancer_sku == "standard" %}
-  "excludeMasterFromStandardLB": {{ azure_exclude_master_from_standard_lb }},
-  "disableOutboundSNAT": {{ azure_disable_outbound_snat }},
+  "excludeMasterFromStandardLB": {{ azure_exclude_master_from_standard_lb | lower }},
+  "disableOutboundSNAT": {{ azure_disable_outbound_snat | lower }},
 {% endif%}
   "loadBalancerSku": "{{ azure_loadbalancer_sku }}"
 }

From 90e6e19403f7d0821a86da6f635b9f9a07c6f0e1 Mon Sep 17 00:00:00 2001
From: Ryler Hockenbury <rhockenbury@users.noreply.github.com>
Date: Wed, 27 Jan 2021 06:53:40 -0500
Subject: [PATCH 7/8] update hashes for 1.18 and 1.19 for kubespray-2.14
 (#7207)

---
 roles/download/defaults/main.yml | 81 ++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)

diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index b20399150b6..2826aa29a62 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -120,10 +120,19 @@ crictl_checksums:
 # Checksums
 kubelet_checksums:
   arm:
+    v1.19.7: 59284dcf4ee3f62475d0d6d1070c64e0e5362280e8d70884513e1e1cee918cb0
+    v1.19.6: 59179a52615cb0811909558425e046cc9c5c051e3b55733b96e40a9aed0967de
+    v1.19.5: 112e247538b889057dd55a35350415494785c6e0f5ad54a273fcab51f4c30805
+    v1.19.4: 51ae27723be81ea63906ec375d652bfe97da312cb35f532d5d483e88aad2fc01
     v1.19.3: 3c0214d2d583440233b6bd0088614fe4fb5748e5b153f2ec96d72d3f8165e214
     v1.19.2: 631e686c34911a40a798817dcff89532c88bb649885f93ec66b339e227ebd974
     v1.19.1: 3985c8d02c1c2f2016fceccd9cc14865e2d047f32c8f0b42aeedcc8450de572e
     v1.19.0: bb433ef7981297bdee6ffc4e23376e8db24a0e47321ebe94bf9d4b9f7a2f0e3b
+    v1.18.15: 27c8d5ea1f837cb0148d829944d46b028ff9d9b9edf12cd39e1cb5f65fd9f41c
+    v1.18.14: e380c14330c4f34b7682a19a493769a4ba7cf9132a5ae9d2ab4109a0ba60973b
+    v1.18.13: ed7e1adef473e4c49cbd4b8f2363239ff08b72f9a2c153f29c4021ef04ca9526
+    v1.18.12: 604db89eca279867aab5b6d0e4619f5577fa85f7817db0620ab5f06806358c2f
+    v1.18.11: 1e5314c30aa480ddb381e003b0e0a8823eccf6b7f49c62718c34afb8b2c833c3
     v1.18.10: 716b8a1971d2f96b28f31a65e6769e2ce60979703d143a67d340c7ce16137db5
     v1.18.9: 24f7559fe5214b5f3b625ab035e29159441e6cfd248befbeb78b63e660fccd23
     v1.18.8: 831f50ea45384884c50395c288d493e75dd134a962dc95261ce122de5e6a17ec
@@ -148,10 +157,19 @@ kubelet_checksums:
     v1.17.1: 0219c940bad3238dfbdf8e4518241d861bbdd8fc93d172cc632c225d7dd57094
     v1.17.0: 75ae6ad8f4a7f2ac3988b37a01c28093f240745d17c1781135d1844057c8ae94
   arm64:
+    v1.19.7: 473016cf1165ab5c705bd29c65451984e48deade6ff96bf861c46e15fc52ba3f
+    v1.19.6: e168d57a0a04abe58db13aaf4f54ba691115d0b1a743d549136794bbecde7dc8
+    v1.19.5: 646373c5337f6f348573bbbef3d92a675fe18d3c54b752805c6d3a9f9fc22957
+    v1.19.4: 8f534dfc4a836d7f434f09c8f726d747204a312eb47f3b874705451c5b04a243
     v1.19.3: 228695df98c5cb8a5f64d1235021f54a482a8e63f5b9b1b2addfc95df9b671ee
     v1.19.2: 86b9336aa7f6215e6f9b387bb82105657668aa8a38b0e0f7c2e647ef45c1b723
     v1.19.1: 143bed1f04cba4e6749a72abf9258d19f31e1a310f94bd041cd30ce62a1f73ff
     v1.19.0: d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf
+    v1.18.15: c74f46e87aae7b9bb590319632fcb106b5efd998046dd47634c89bbb875d9fb6
+    v1.18.14: b4ca12f6b1a5f7c838831eb3c9152ccf0f53b83a3492c812da6ac680959a4801
+    v1.18.13: 953b7f9c70d04be0cf69745ab30f676375d93272362c97bb3cd527d6e27b38e4
+    v1.18.12: 2593c564811f38202fdde351c8ee8d3cfac2dd7c52e42b393157ac6fa1569cbd
+    v1.18.11: d595561315eb7fd270363464a2e36c702684086410000212333478761b8c13c1
     v1.18.10: 1490550560b9afcb6e74d5bd69d61ae60dabe466e05e0408da48f17b4ccd91b4
     v1.18.9: 21b0fb4682deea19be3ac160403db9858dc9d02b101d60eb6fc22a86523ec434
     v1.18.8: d36e2d656bad232e8b48b19c948164ee3966669f4566cf5ea43ca22f6eed1aa5
@@ -176,10 +194,19 @@ kubelet_checksums:
     v1.17.1: c773512ade5da3188ed4c312d5ba01bfbf3f376f6e580e5b074827a5b25450aa
     v1.17.0: b1a4a2325383854a69ec768e7dc00f69378d3ccbc554859d910bf5b582264ea2
   amd64:
+    v1.19.7: d8b296825f6dd7a17287b73cd6604d32210abbba86c88fb68c1b1c5016906c54
+    v1.19.6: 8162fa58f7fbb9b1f07b2b666c9759ad5c30950bc9f6f2084de1c0d9bc64a4c0
+    v1.19.5: b64dbc4a1a47d21be0a67f7d0050eebc02dd610c31175865da08e3a59e0834af
+    v1.19.4: 078cd88f521092fb1b9e9596f64ab72de5af3d4935e22d8421ad5e45c9729b1d
     v1.19.3: daa02a34efd936bf9940d9c52fe24c299fc36ba4b31a051968efb3652f289fa9
     v1.19.2: 7ff6d3663e8de0c654909e7a279e386286aa7ed3fc262d021bed77c92d62780f
     v1.19.1: 2ca2a3104d4cce26db128e3a0b7a042385df4f2c51bdbe740e067fdfaa2fcdd1
     v1.19.0: 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab
+    v1.18.15: 2d079bc9f7ac2c2d1f86df842df55f2ec8e74e01edc347994ccf1a054b9e6077
+    v1.18.14: 1876c3aad83beeea1bc949fe6121d0d0d9002a0e026c15ccb568d8c5e748fba6
+    v1.18.13: dbdecaec19da684055242e2684ec8a2bff1674adf9ae8b8ed87f9cb46101a87f
+    v1.18.12: 928900515d7f4bb35b80bb39ef934d133c2917aa9d65ec33f00304079f474305
+    v1.18.11: 3e5c5a784b8be476248e31819204e02779d5444cbe3c77789562e34a86c11c0b
     v1.18.10: 8daecd339993342c0693b6cb8a8e053d4a21d2d829389cc7ab823f52ea0589a1
     v1.18.9: 5028b6548e8838e1e0851f10e8bd8d9a6ef1693e3f1dac09f7d50c4c2873f20b
     v1.18.8: a4116675ac52bf80e224fba8ff6db6f2d7aed192bf6fffd5f8e4d5efb4368f31
@@ -205,10 +232,19 @@ kubelet_checksums:
     v1.17.0: c2af77f501c3164e80171903028d35c632366f53dec0c8419828d4e55d86146f
 kubectl_checksums:
   arm:
+    v1.19.7: 11d49f87bf8b557066c339eea4775b32dd463fc61ce24b24744f82cb412e9277
+    v1.19.6: fde28a2bff5892e461d5c72c5391da5eef20450a5a074bbbfce4f476687dac95
+    v1.19.5: 10409729115f66e32263dfa8a320b74ef1e5845a9103470d3d18095ca4c1dc80
+    v1.19.4: 4ca2dc66fb5ea5f71afbb1f769e7eae237da4c9287c875da935a6b9f15619112
     v1.19.3: fb611ff64139bc8712fe93497f2419c236d62c5f689e1cb4cc68037fda698f82
     v1.19.2: c8cad74a586dfee41436ce866c91d79c9d2053468eccde9fed976cdf32b7849f
     v1.19.1: e63bbf3161c49d60e46ffaf7d3cfd689834516205be1be881d2e652115535e93
     v1.19.0: 5885bb723a55ab95e8687e0ad52620ce4c08f76fd20c0973f5cd19c753b513c8
+    v1.18.15: 952530dd6b272eed932e49a29625e20303b9621ba36c1cc0394476d483f6860a
+    v1.18.14: b2b88ee6bea8ee17dd1c7687add53c9db5139abb7013ded77050d57b62070aa7
+    v1.18.13: f3e9a4786a774441ee448edb6f842f9c6825f12245b5e4ee5ffe8b2ab1c85058
+    v1.18.12: 4380ddb37d49e1199ebaa1f2785bd81ae22ffac15ad470d48f7308e9bf131b26
+    v1.18.11: b357d042224f13be5cb7f11bd25bcd735ef2fee8bcdd43653598d616ac8bb1f0
     v1.18.10: 3d5b7cb1c54d5e9dec157a512d2d21dddc6b9fd5b9a0b8df9493553871d21668
     v1.18.9: 8f49ade7875aaca82f7471901963796815b786d5437e6af0ae4d6d784dc92c08
     v1.18.8: 21769e01e17f3809d0e9188a88a71fb1f479dfeeb22590e56006d5dbb3689605
@@ -233,10 +269,19 @@ kubectl_checksums:
     v1.17.1: a1e580e9140536c4a370c207ee66481cfe8d8876dc9021755a9d20232a97033d
     v1.17.0: 594b3e2f89dca09d82b176b51bf6c8c0fa524ed209c14ec915c9b36fa876601d
   arm64:
+    v1.19.7: a0f58222e8d62b86b1148746531fa0c3b91fa208586cb068883550fc3dcd498b
+    v1.19.6: 828ddf7183c647306a2432ff098a5b22fd6c68c069cb7047ffb1e2b2a967ef83
+    v1.19.5: eaf26ca30f1696e72754d86aeda1886287127295809a400821f8d0ffa6329359
+    v1.19.4: 334ca7ce181d8b83e100b1ad4347af93bee6f261d792e9f52d9769095e970919
     v1.19.3: a4f2e2dbdcead30eed5aa47468e669b9574fd99457b860679eba84e1cb9cf863
     v1.19.2: a460f918c03e5cd916f4ac28da137596232e344cc0755d4ceb222fc4cd203e09
     v1.19.1: 332bbdb4560f9b7fcbb96c8f2cebbc4996e409384ca07510e5c5990998158c20
     v1.19.0: d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f
+    v1.18.15: 6b4a63df325cdb523f16ffd8799745a8fdf979ef89e228c309a715671b6aa984
+    v1.18.14: ac4014f7d9001375cb600a15d77e90eb6a20053afac82f167f4b7732aa073388
+    v1.18.13: 8e5271e95442e373df1c67473484e387086f344a2e1445ee9f5a878ca7f4442c
+    v1.18.12: 3cae828a3dca46b12284839e4273a1cfcf35177937ca5aa5aa49193d6eb0649b
+    v1.18.11: fefd809b8749f2884ec1a99acf987286b487eac90c98145aef8ac866d0ceae99
     v1.18.10: 394fa475f7688778eeeecb31e01acfae4cc37a72926d9bf33290c968e6dc037a
     v1.18.9: 9f466ff8d40097914a1ded0288ef8b9eb6f4ad39a9990cb2c0f8d1a710013a4f
     v1.18.8: 9046c4086528427462544e1a6dcbe709de4d7ae44d1a155375de330fecd067b1
@@ -261,10 +306,19 @@ kubectl_checksums:
     v1.17.1: 4208be10e2c12b67e71219cd39b0b2ab065d4ec1b26e19c5da88cb8ebc64ea2f
     v1.17.0: cba12bfe0ee447b06f00813d7d4ba3fbdbf5116eccc4d3291987044f2d6f93c2
   amd64:
+    v1.19.7: d46eb3bbe2575e5b6bedbc6d3519424b4f2f57929d7da1ef7e11c09068f37297
+    v1.19.6: d8a46600bcdcd01f39c11e609e8277975f04c0593f79b2a7b5c67646e1c792d8
+    v1.19.5: 5f5a0bebde8a8782b2d9de9bd314dce722f9f960bee090b121ea5ac38cf4377d
+    v1.19.4: 7df333f1fc1207d600139fe8196688303d05fbbc6836577808cda8fe1e3ea63f
     v1.19.3: 84eeb8237448e4f431fef0f0ec0ba8b07558d8e52d5a7e89b4ae64dadcffbe66
     v1.19.2: f51adfe7968ee173dbfb3dabfc10dc774983cbf8a3a7c1c75a1423b91fda6821
     v1.19.1: da4de99d4e713ba0c0a5ef6efe1806fb09c41937968ad9da5c5f74b79b3b38f5
     v1.19.0: 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f
+    v1.18.15: eb5a5dd0a72795942ab81d1e4331625e80a90002c8bb39b2cb15aa707a3812c6
+    v1.18.14: 8c924c1fdf743c2a3bf0edbd4333f54c1bce64871abc1a729243321d99b567d4
+    v1.18.13: 8914a4529aaa5f358c663c03bc2cb741e5667f8142e37435689a851647b7697f
+    v1.18.12: ea2f6d0542b7bd259ccf7dd139fdad20767f7c5bf94c19c50df227396a08d76d
+    v1.18.11: 1bdab3fd6aecf1ab40fa9076e8ac487cd028a16f911da2d25d121033f6f61de0
     v1.18.10: b25c445cb36ab168de590c13b50bced4e7877ed15ca899a261053b4ba3ba1456
     v1.18.9: 6a68756a2d3d04b4d0f52b00de6493ba2c1fcb28b32f3e4a0e99b3d9f6c4e8ed
     v1.18.8: a076f5eff0710de94d1eb77bee458ea43b8f4d9572bbb3a3aec1edf0dde0a3e7
@@ -290,10 +344,19 @@ kubectl_checksums:
     v1.17.0: 6e0aaaffe5507a44ec6b1b8a0fb585285813b78cc045f8804e70a6aac9d1cb4c
 kubeadm_checksums:
   arm:
+    v1.19.7: 48722b7c93d18a13e734200de202912a324769139d0d434ff1dfff82af164814
+    v1.19.6: e3ecde2f2eccb177c0e4b8c6bd19ae471bc19977a8f812cb17094743d42b5b6e
+    v1.19.5: b034594ebe9096f27b3bc5a0b1a98baed9d54ba1457d3d13208f0d490db39000
+    v1.19.4: 9b83c9dcda775e9b5a20d7254218e8b70cea8a37266d254b6289faa1d5602cd9
     v1.19.3: 522358c8596d10cac1a04a9e52f0ae59a1c06ca122292429d36773e7f6ad0a01
     v1.19.2: effc35d1e3ab01ac80185ff9f7ca5afabbb94f5f91d7326b04b09e903315647d
     v1.19.1: 0e910cf9c771976f6eb079098ad428f3e99080624f478e3d71b670005a7c3651
     v1.19.0: 62fca8b373f8c63409bcfcb7c585f8de882a8a119d88d39666e1ab3a11be188b
+    v1.18.15: b242890123a5ecc574157a959475c903eeb14ed17b4578902a17b8d994d4c401
+    v1.18.14: 7e5d4beedcaf13c0076f03c9232464946faa061effd5db8c7705d317a4ee6e95
+    v1.18.13: 3785825c1b9a1fbb90abc00077d9ccd43610b147302c29501b9ce12959fb13bf
+    v1.18.12: 020d1adba1c339fa390fac18cf839f70d52d981b33b62b7b2f1ca4166c263bd0
+    v1.18.11: 0647721b5a4a4b6932f2ce43f8a35f3cf0d875b598a73eee209b8955beaede40
     v1.18.10: 49f53573bdefd4ed37376800119b082666d03d5657d5886a4caa35e63a11d658
     v1.18.9: 026cd1ec3b75703994254ae44998a544f46723b424775218f90c07754bb42bb6
     v1.18.8: 52ec1a5d8a6826762c112d55734e35cf895a02e746b8d6ca4a9c942289aab077
@@ -318,10 +381,19 @@ kubeadm_checksums:
     v1.17.1: 501d1bacb863713dd9d0101d0021b0227869c4b1b9e903f6498333c613d384e1
     v1.17.0: 5fcf1234d89bc2a364c53b76b36134fc57278b456138d93c278805f2c9b186f1
   arm64:
+    v1.19.7: 43615e65092de34dcaf34725d8adc0af91b660fa54a885421fdb29f743478911
+    v1.19.6: 082ceac5f542cb49a62cf47212bf1ea9dbb15d1831126f526d15e429b4f0687d
+    v1.19.5: 8c4363e910f0070d597f3e5b9a7b8741e1ef20778a8dddc15ac47bf02389d964
+    v1.19.4: 1357ce6314a2b4a2303a96c33c996e7e9e8269cf4cc464557e7fa09f9a6ade34
     v1.19.3: c398c23019f988514ac0f2c1e32a388cf11ca9d48634530092dbf54d9e00eaa6
     v1.19.2: b6900a44558ba1a0a364406e2072163f5fc561030da97045e1403cdc69ff2682
     v1.19.1: dcdabd2fdec9b4dd8febd1625e1fbbe15362919041e5f4ce3aab629e4aea7540
     v1.19.0: db1c432646e6e6484989b6f7191f3610996ac593409f12574290bfc008ea11f5
+    v1.18.15: d6d4d9b8e4992c218ff6185da7928289c9938796d5c08a7427625a563c74a075
+    v1.18.14: e4c1aaed946dd38ca6d9fdef0ef333b4d488a25810aa6f8a98de1b9dd6d47db3
+    v1.18.13: 54d035ea9356b2f7f146d7e287aba128d3a0c594e0f0112f6ccba1d0294770c9
+    v1.18.12: d4cbd57c504000e7eeb68dfcb3b5e08625baa6a60b970b053a7234907b173272
+    v1.18.11: c36aec51851c9b3cbce61fbea26a70af2923b339c40b92973f4a0e8b1c584190
     v1.18.10: dc4a2daa3bf3e652fc7a81f5b449752c08e6a91e27aa1bbffad7ade35508a77b
     v1.18.9: c17e29b8cec1892b6cd72aed1af6d9abfd39816c222d3cc5c97c6637a284162d
     v1.18.8: 71f6d95f165a9e8066c6f299217af779829ab3d798f6130caf6daa4784dc0464
@@ -346,10 +418,19 @@ kubeadm_checksums:
     v1.17.1: c640eb50406962628ac6e31fd840506a360b5d9c57d14007d0eaada28c49d64f
     v1.17.0: 0b94d1ace240a8f9995358ca2b66ac92072e3f3cd0543275b315dcd317798546
   amd64:
+    v1.19.7: c63ef1842533cd7888c7452cab9f320dcf45fc1c173e9d40abb712d45992db24
+    v1.19.6: 6204d9f16554480fe64a09e9efef31559f1da623fb34492a9a18b085afac876a
+    v1.19.5: 00a2eb2b919661b46cd845c5645e3258806f331f1ea8015e136e733c8c5dd515
+    v1.19.4: e1977c6ba7acc7b2b3896202c93a73dbdf1c649b93843350e0f10172e26cf59d
     v1.19.3: 0a7581fdebe05fb101ce30d4e1f85e865e18f5c034e4f7cc785c786e861f9be2
     v1.19.2: 377dbf06469709aafb7a9a6c925e890bf48727b905455c2a81fee28727716f2f
     v1.19.1: d5afcf4ff916b4c6810f10f76176c73238f0854b7748b2cde8afbd91de65a3c9
     v1.19.0: 88ce7dc5302d8847f6e679aab9e4fa642a819e8a33d70731fb7bc8e110d8659f
+    v1.18.15: 8a5be9e04343e0ac10320455b32a78e5ffc60f450c5c0a11914edeb86ca178d7
+    v1.18.14: d6143cd822218daa5faf583c9b8b862c609e66052232e3d3d23c72957fdae341
+    v1.18.13: cad917c516bc326ecc17c50bf81b1c0bb153a5090206ccf88d48c23839941d08
+    v1.18.12: 6a0d1aa32adabfe84a1b12ab71b570631dd4db24e2747830f8209fb7d94ecca6
+    v1.18.11: a4b7caa7fbe1de94834b3715567956527643cdbb0e3a4688a8788ac6861dc26f
     v1.18.10: 9bf46e5276bc14d42d6dcf05ac507bb3236ce8dc0fa21aad985d9328c377c18d
     v1.18.9: 3f7f61e0fe3de43f5b345343f85d7ba5145737efb80974baa6076965f3a6963e
     v1.18.8: 27c8f4d4398d57762998b157d35802a36a7ea9b2b6f9a363c397a9d65b4f3c89

From c3814bb258217d38f3a0da2b501fdeeae92c1c58 Mon Sep 17 00:00:00 2001
From: Rick Haan <RickHaan@users.noreply.github.com>
Date: Fri, 29 Jan 2021 11:33:40 +0100
Subject: [PATCH 8/8] Check kube-apiserver up on all masters before upgrade
 (#7193) (#7197)

Only checking the kubernetes api on the first master when upgrading is not enough.
Each master needs to be checked before it's upgrade.

Signed-off-by: Rick Haan <rickhaan94@gmail.com>
---
 roles/kubernetes/master/tasks/kubeadm-upgrade.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/kubernetes/master/tasks/kubeadm-upgrade.yml b/roles/kubernetes/master/tasks/kubeadm-upgrade.yml
index cd6112b2d0a..b9c54deabb0 100644
--- a/roles/kubernetes/master/tasks/kubeadm-upgrade.yml
+++ b/roles/kubernetes/master/tasks/kubeadm-upgrade.yml
@@ -3,7 +3,7 @@
   uri:
     url: "https://{{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}/healthz"
     validate_certs: false
-  when: inventory_hostname == groups['kube-master']|first
+  when: inventory_hostname == groups['kube-master']
   register: _result
   retries: 60
   delay: 5