From 82756cf73d87305033d60b12b1bd1718ecf9e932 Mon Sep 17 00:00:00 2001 From: Latha Sivakumar Date: Thu, 21 Jan 2021 14:16:53 -0500 Subject: [PATCH 1/4] (#62) Pre_Install_Check: Add additional messages to report when All or Some Node(s) are not in READY state --- .../library/pre_install_check.py | 18 +++- .../library/pre_install_check_permissions.py | 92 +++++++++++++++---- .../library/pre_install_utils.py | 10 +- .../library/utils/viya_constants.py | 1 + .../report_template_viya_pre_install_check.j2 | 4 +- .../json_data/azure_nodes_no_master.json | 16 ++-- .../test/test_pre_install_report.py | 15 ++- 7 files changed, 117 insertions(+), 39 deletions(-) diff --git a/pre_install_report/library/pre_install_check.py b/pre_install_report/library/pre_install_check.py index 8a492dc..55ff145 100644 --- a/pre_install_report/library/pre_install_check.py +++ b/pre_install_report/library/pre_install_check.py @@ -73,6 +73,7 @@ def __init__(self, sas_logger: ViyaARKLogger, viya_kubelet_version_min, viya_min self._viya_min_aggregate_worker_memory: Text = viya_min_aggregate_worker_memory self._calculated_aggregate_allocatable_memory = None self._workers = 0 + self._aggregate_nodeStatus_failures = 0 def _parse_release_info(self, release_info): """ @@ -609,11 +610,15 @@ def _check_kubelet_errors(self, global_data, aggregate_kubelet_failures): return: updated global data about worker nodes retrieved """ aggregate_kubelet_data = {} - aggregate_kubelet_data.update({'aggregate_kubelet_failures': str(aggregate_kubelet_failures)}) + node_status_msg = "" + if self._aggregate_nodeStatus_failures > 0: + node_status_msg = " Check Node(s). All Nodes NOT in Ready Status." \ + + ' Issues Found: ' + str(self._aggregate_nodeStatus_failures) + aggregate_kubelet_data.update({'aggregate_kubelet_failures': node_status_msg}) if aggregate_kubelet_failures > 0: aggregate_kubelet_data.update({'aggregate_kubelet_failures': 'Check Kubelet Version on nodes.' + - ' Issues Found: ' + str(aggregate_kubelet_failures)}) + ' Issues Found: ' + str(aggregate_kubelet_failures) + '.' + node_status_msg}) global_data.append(aggregate_kubelet_data) return global_data @@ -734,6 +739,15 @@ def evaluate_nodes(self, nodes_data, global_data, cluster_info, quantity_): alloc_memory = str(node['allocatableMemory']) total_memory = total_memory + quantity_(str(node['memory'])) total_allocatable_memory = total_allocatable_memory + quantity_(alloc_memory) + # nodestatus = False + try: + nodeReady = str(node['Ready']) + if nodeReady == "True": + pass + else: + self._aggregate_nodeStatus_failures += 1 + except KeyError: + node['Ready'] = viya_constants.KEY_NOT_FOUND if node['worker']: total_cpu_cores = total_cpu_cores + alloc_cpu_cores diff --git a/pre_install_report/library/pre_install_check_permissions.py b/pre_install_report/library/pre_install_check_permissions.py index 95d0690..83c3454 100644 --- a/pre_install_report/library/pre_install_check_permissions.py +++ b/pre_install_report/library/pre_install_check_permissions.py @@ -65,6 +65,8 @@ def __init__(self, params): self.ingress_data[viya_constants.INGRESS_CONTROLLER] = self.ingress_controller self.ingress_file = "hello-ingress.yaml" self._storage_class_sc: List[KubernetesResource] = None + self._sample_deployment = 0 + self._sample_output = "" def _set_results_cluster_admin(self, resource_key, rc): """ @@ -73,7 +75,9 @@ def _set_results_cluster_admin(self, resource_key, rc): """ if rc == 1: self.cluster_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS - self.cluster_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] = viya_constants.INSUFFICIENT_PERMS + self.cluster_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] = \ + viya_constants.INSUFFICIENT_PERMS + ". Check Logs." + else: self.cluster_admin_permission_data[resource_key] = viya_constants.ADEQUATE_PERMS @@ -82,12 +86,41 @@ def _set_results_namespace_admin(self, resource_key, rc): Set permissions status for specified resource/verb with namespace admin role """ - if rc == 1: - self.namespace_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS - self.namespace_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] \ - = viya_constants.INSUFFICIENT_PERMS + sample_keys = [viya_constants.PERM_DEPLOYMENT] + deployment_keys = [viya_constants.PERM_DELETE + viya_constants.PERM_DEPLOYMENT, + viya_constants.PERM_SERVICE, + viya_constants.PERM_DELETE + viya_constants.PERM_SERVICE, + viya_constants.PERM_INGRESS, + viya_constants.PERM_DELETE + viya_constants.PERM_INGRESS, + viya_constants.PERM_REPLICASET, + viya_constants.PERM_CREATE + viya_constants.PERM_ROLE, + viya_constants.PERM_CREATE + viya_constants.PERM_ROLEBINDING, + viya_constants.PERM_CREATE + viya_constants.PERM_SA, + viya_constants.PERM_DELETE + viya_constants.PERM_ROLE, + viya_constants.PERM_DELETE + viya_constants.PERM_ROLEBINDING, + viya_constants.PERM_DELETE + viya_constants.PERM_SA + ] + if rc != 0: + self.logger.debug("resource_key = {}, sample_deployment = {} ".format(str(resource_key), + str(self._sample_deployment))) + if self._sample_deployment != 0: + if resource_key in deployment_keys: + self.namespace_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS + if resource_key in sample_keys: + self.namespace_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS + \ + ". Sample Deployment Check failed! " + \ + "Ensure Node(s) Status is Ready. " + \ + "Check Permissions in specified namespace. " \ + + self._sample_output + + else: + self.namespace_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS + self.namespace_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] = \ + viya_constants.INSUFFICIENT_PERMS + ". Check Logs." else: self.namespace_admin_permission_data[resource_key] = viya_constants.ADEQUATE_PERMS + # self.namespace_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] = \ + # viya_constants.ADEQUATE_PERMS def _get_pvc(self, pvc_name, key): """ @@ -284,20 +317,34 @@ def check_sample_application(self): rc = self.utils.deploy_manifest_file(viya_constants.KUBECTL_APPLY, 'hello-application.yaml') - # self._set_results_namespace_admin(viya_constants.PERM_DEPLOYMENT, rc) - # self._set_results_namespace_admin(viya_constants.PERM_SERVICE, rc) if rc == 0: - rc = self.utils.do_cmd(" rollout status deployment.v1.apps/hello-world ") + rc, sample_output = self.utils.do_cmd(" rollout status deployment.v1.apps/hello-world --timeout=180s") + # You can check if a Deployment has completed by using kubectl rollout status. + # If the rollout completed successfully, kubectl rollout status returns a zero exit code. + + if rc != 0: + self._sample_deployment = 2 + self._sample_output = sample_output + self._set_results_namespace_admin(viya_constants.PERM_DEPLOYMENT, rc) + self._set_results_namespace_admin(viya_constants.PERM_SERVICE, rc) + return 2 self._set_results_namespace_admin(viya_constants.PERM_DEPLOYMENT, rc) self._set_results_namespace_admin(viya_constants.PERM_SERVICE, rc) if rc == 0: - rc = self.utils.do_cmd(" scale --replicas=2 deployment/hello-world ") + rc, sample_output = self.utils.do_cmd(" scale --replicas=2 deployment/hello-world ") + if rc != 0: + self._sample_deployment = 3 + self._set_results_namespace_admin(viya_constants.PERM_REPLICASET, rc) + return 3 + else: + self._sample_deployment = 1 + self._set_results_namespace_admin(viya_constants.PERM_DEPLOYMENT, rc) + self._set_results_namespace_admin(viya_constants.PERM_SERVICE, rc) - if rc == 0: - self._set_results_namespace_admin(viya_constants.PERM_REPLICASET, rc) + return 1 def check_sample_service(self): """ @@ -396,8 +443,9 @@ def check_delete_sample_application(self): rc = self.utils.deploy_manifest_file(viya_constants.KUBECTL_DELETE, 'hello-application.yaml') self._set_results_namespace_admin(viya_constants.PERM_DELETE + viya_constants.PERM_DEPLOYMENT, rc) + self._set_results_namespace_admin(viya_constants.PERM_DELETE + viya_constants.PERM_SERVICE, rc) - rc = self.utils.do_cmd(" wait --for=delete pod -l app=hello-world-pod --timeout=12s ") + self.utils.do_cmd(" wait --for=delete pod -l app=hello-world-pod --timeout=12s ") def check_delete_sample_service(self): """ @@ -443,12 +491,13 @@ def check_deploy_crd(self): def check_rbac_role(self): """ Check if RBAC is enabled in specified namespace - Create the Role and Rolebinding for the custome resource access with specified namespace. Set the + Create the Role and Rolebinding for the custom resource access with specified namespace. Set the permissions status in the namespace_admin_permission_data dict object. """ found = self.utils.get_rbac_group_cmd() - + self.logger.debug("get_rbace_group_cmd found = {}, sample_deployment = {}" + .format(str(found), str(self._sample_deployment))) if found: rc = self.utils.deploy_manifest_file(viya_constants.KUBECTL_APPLY, 'viya-role.yaml') @@ -463,7 +512,14 @@ def check_rbac_role(self): 'viya-rolebinding.yaml') self._set_results_namespace_admin(viya_constants.PERM_CREATE + viya_constants.PERM_ROLEBINDING, rc) else: + self.logger.debug("sample_deployment = {}".format(str(self._sample_deployment))) self.namespace_admin_permission_aggregate["RBAC Checking"] = viya_constants.PERM_SKIPPING + self._set_results_namespace_admin(viya_constants.PERM_CREATE + viya_constants.PERM_ROLE, + int(self._sample_deployment)) + self._set_results_namespace_admin(viya_constants.PERM_CREATE + viya_constants.PERM_SA, + int(self._sample_deployment)) + self._set_results_namespace_admin(viya_constants.PERM_CREATE + viya_constants.PERM_ROLEBINDING, + int(self._sample_deployment)) def check_rbac_delete_role(self): """ @@ -495,14 +551,14 @@ def check_get_custom_resource(self, namespace): if not allowed: rc1 = 1 - self._set_results_namespace_admin_crd(viya_constants.PERM_CREATE + viya_constants.PERM_CR + " with RBAC " - + viya_constants.PERM_SA + " resp: = " + str(allowed), rc1) + self._set_results_namespace_admin_crd(viya_constants.PERM_CREATE + viya_constants.PERM_CR_RBAC + + viya_constants.PERM_SA, rc1) allowed: bool = self.utils.can_i(' delete viyas.company.com --as=system:serviceaccount:' + namespace + ':crreader ') if allowed: rc2 = 1 - self._set_results_namespace_admin_crd(viya_constants.PERM_DELETE + viya_constants.PERM_CR + " with RBAC " - + viya_constants.PERM_SA + " resp: = " + str(allowed), rc2) + self._set_results_namespace_admin_crd(viya_constants.PERM_DELETE + viya_constants.PERM_CR_RBAC + + viya_constants.PERM_SA, rc2) def check_delete_custom_resource(self): """ diff --git a/pre_install_report/library/pre_install_utils.py b/pre_install_report/library/pre_install_utils.py index de9dfde..c27718d 100644 --- a/pre_install_report/library/pre_install_utils.py +++ b/pre_install_report/library/pre_install_utils.py @@ -77,12 +77,12 @@ def do_cmd(self, test_cmd): self.logger.info("cmd {} rc = 0".format(test_cmd)) self.logger.debug("cmd {} rc = 0 response {}".format(test_cmd, str(data))) - return 0 + return 0, str(data) except CalledProcessError as e: data = e.output self.logger.error("do_cmd " + ' rc = ' + str(e.returncode) + test_cmd + ' data = ' + str(data)) - return e.returncode + return e.returncode, str(data) def get_rbac_group_cmd(self): """ @@ -91,14 +91,14 @@ def get_rbac_group_cmd(self): cmd: kubectl command to retrieve api_resources return: True if both Role and RoleBinding kinds have an api_group """ - role = None - rolebinding = None + role: bool = None + rolebinding: bool = None try: data: KubernetesApiResources = self._kubectl.api_resources(False) role = data.get_api_group("Role") rolebinding = data.get_api_group("RoleBinding") except CalledProcessError as e: - self.logger.exception("get_rbac_group_cmd rc {}" + str(e.returncode)) + self.logger.exception("get_rbac_group_cmd rc {} ".format(str(e.returncode))) return False if role is None: return False diff --git a/pre_install_report/library/utils/viya_constants.py b/pre_install_report/library/utils/viya_constants.py index ebe5b9c..6940401 100644 --- a/pre_install_report/library/utils/viya_constants.py +++ b/pre_install_report/library/utils/viya_constants.py @@ -50,3 +50,4 @@ PERM_ROLEBINDING = "RoleBinding" PERM_SA = "Service Account" PERM_CLASS = "PreInstallUtils" +PERM_CR_RBAC = "Custom Resource with RBAC " diff --git a/pre_install_report/templates/report_template_viya_pre_install_check.j2 b/pre_install_report/templates/report_template_viya_pre_install_check.j2 index ecd072d..7d469a0 100644 --- a/pre_install_report/templates/report_template_viya_pre_install_check.j2 +++ b/pre_install_report/templates/report_template_viya_pre_install_check.j2 @@ -189,6 +189,7 @@ Kubelet Version Container Runtime Kernel Version + Node Status Issues {% for node in nodes_data %} @@ -197,6 +198,7 @@ {{node.kubeletversion}} {{node.containerRuntimeVersion}} {{node.kernelVersion}} + {{node.Ready}} {{node.error.kubeletversion}} {% endfor %} @@ -355,7 +357,7 @@
-

Namespace Admin Permissions: {{ namespace_admin_permission_aggregate }}

+

Namespace Admin Permissions - {{ namespace_admin_permission_aggregate }}

diff --git a/pre_install_report/test/test_data/json_data/azure_nodes_no_master.json b/pre_install_report/test/test_data/json_data/azure_nodes_no_master.json index 4f0dc08..15f468b 100644 --- a/pre_install_report/test/test_data/json_data/azure_nodes_no_master.json +++ b/pre_install_report/test/test_data/json_data/azure_nodes_no_master.json @@ -112,7 +112,7 @@ "lastTransitionTime": "2020-09-02T20:10:38Z", "message": "kubelet is posting ready status. AppArmor enabled", "reason": "KubeletReady", - "status": "True", + "status": "Unknown", "type": "Ready" } ], @@ -651,7 +651,7 @@ "lastTransitionTime": "2020-09-02T20:10:08Z", "message": "kubelet is posting ready status. AppArmor enabled", "reason": "KubeletReady", - "status": "True", + "status": "Unknown", "type": "Ready" } ], @@ -816,7 +816,7 @@ "lastTransitionTime": "2020-09-02T20:09:34Z", "message": "kubelet is posting ready status. AppArmor enabled", "reason": "KubeletReady", - "status": "True", + "status": "Unknown", "type": "Ready" } ], @@ -993,7 +993,7 @@ "lastTransitionTime": "2020-09-02T20:09:38Z", "message": "kubelet is posting ready status. AppArmor enabled", "reason": "KubeletReady", - "status": "True", + "status": "Unknown", "type": "Ready" } ], @@ -1175,7 +1175,7 @@ "lastTransitionTime": "2020-09-02T20:09:54Z", "message": "kubelet is posting ready status. AppArmor enabled", "reason": "KubeletReady", - "status": "True", + "status": "Unknown", "type": "Ready" } ], @@ -1326,7 +1326,7 @@ "lastTransitionTime": "2020-09-02T20:09:42Z", "message": "kubelet is posting ready status. AppArmor enabled", "reason": "KubeletReady", - "status": "True", + "status": "Unknown", "type": "Ready" } ], @@ -1469,7 +1469,7 @@ "lastTransitionTime": "2020-09-02T20:06:29Z", "message": "kubelet is posting ready status. AppArmor enabled", "reason": "KubeletReady", - "status": "True", + "status": "Unknown", "type": "Ready" } ], @@ -1612,7 +1612,7 @@ "lastTransitionTime": "2020-09-02T20:06:46Z", "message": "kubelet is posting ready status. AppArmor enabled", "reason": "KubeletReady", - "status": "True", + "status": "Unknown", "type": "Ready" } ], diff --git a/pre_install_report/test/test_pre_install_report.py b/pre_install_report/test/test_pre_install_report.py index 3f57e39..d31055c 100644 --- a/pre_install_report/test/test_pre_install_report.py +++ b/pre_install_report/test/test_pre_install_report.py @@ -184,7 +184,7 @@ def test_get_nested_nodes_info(): ' Issues Found: 1' total_allocatable_memoryG = vpc.get_calculated_aggregate_memory() # quantity_("62.3276481628418 Gi").to('G') assert str(round(total_allocatable_memoryG.to("G"), 2)) == '66.92 G' - assert global_data[4]['aggregate_kubelet_failures'] in '1, Check Kubelet Version on nodes. Issues Found: 1' + assert global_data[4]['aggregate_kubelet_failures'] in 'Check Kubelet Version on nodes. Issues Found: 1.' template_render(global_data, configs_data, storage_data, 'nested_nodes_info.html') @@ -227,7 +227,7 @@ def test_get_nested_millicores_nodes_info(): total_allocatable_memoryG = vpc.get_calculated_aggregate_memory() assert str(round(total_allocatable_memoryG.to("G"), 2)) == '66.92 G' - assert global_data[4]['aggregate_kubelet_failures'] in '2, Check Kubelet Version on nodes. Issues Found: 2' + assert global_data[4]['aggregate_kubelet_failures'] in 'Check Kubelet Version on nodes. Issues Found: 2.' template_render(global_data, configs_data, storage_data, 'nested_millicores_nodes_info.html') @@ -456,6 +456,9 @@ def test_azure_multi_get_nested_nodes_info(): global_data = [] cluster_info = "Kubernetes master is running at https://node3:6443\n" + for node in nodes_data: + assert node['Ready'] in 'True' + global_data = vpc.evaluate_nodes(nodes_data, global_data, cluster_info, quantity_) pprint.pprint(global_data) for nodes in global_data: @@ -469,7 +472,7 @@ def test_azure_multi_get_nested_nodes_info(): def test_azure_worker_nodes(): - + viya_kubelet_version_min = 'v1.17.0' vpc = createViyaPreInstallCheck(viya_kubelet_version_min, viya_min_worker_allocatable_CPU, viya_min_aggregate_worker_CPU_cores, @@ -489,7 +492,7 @@ def test_azure_worker_nodes(): global_data = [] cluster_info = "Kubernetes master is running at https://node3:6443\n" - + issues_found = 8 global_data = vpc.evaluate_nodes(nodes_data, global_data, cluster_info, quantity_) pprint.pprint(global_data) for nodes in global_data: @@ -497,7 +500,9 @@ def test_azure_worker_nodes(): 'Expected: 12, Calculated: 141.56, Issues Found: 0' assert global_data[3]['aggregate_memory_failures'] in 'Expected: 56G, Calculated: 727.85 G, ' \ 'Issues Found: 0' - assert global_data[4]['aggregate_kubelet_failures'] in '0, Check Kubelet Version on nodes.' + assert global_data[4]['aggregate_kubelet_failures'] in 'Check Kubelet Version on nodes. Issues Found: 10. ' \ + 'Check Node(s). All Nodes NOT in Ready Status. ' \ + 'Issues Found: ' + str(issues_found) template_render(global_data, configs_data, storage_data, 'azure_nodes_no_master.html') From 33a268e57ba492b129d31cda527f0de66cf75f8f Mon Sep 17 00:00:00 2001 From: Latha Sivakumar Date: Mon, 25 Jan 2021 16:05:04 -0500 Subject: [PATCH 2/4] (#62) Pre_Install_Check: Add additional messages to report when All or Some Node(s) are not in READY state --- pre_install_report/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index c8291ef..c127b2a 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -58,6 +58,10 @@ $ export INGRESS_HOST=externalIP=$(kubectl -n get service get service -o jsonpath='{.spec.ports[?(@.name=="http")].port}') $ export INGRESS_HTTPS_PORT=$(kubectl -n get service -o jsonpath='{.spec.ports[?(@.name=="https")].port}') ``` +The command to determin host name may be slightly different with Amazon EKS: +``` +$ export INGRESS_HOST=externalIP=$(kubectl -n get service -o jsonpath='{.status.loadBalancer.ingress[*].hostname}') +``` Use the values gathered on the command line for http or https as appropriate for your deployment: From f8c5b59c15b8985c487c48b6e2e6dc05824cca7c Mon Sep 17 00:00:00 2001 From: Latha Sivakumar Date: Mon, 25 Jan 2021 16:16:43 -0500 Subject: [PATCH 3/4] (#62) Pre_Install_Check: Add additional messages to report when All or Some Node(s) are not in READY state --- pre_install_report/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index c127b2a..2978c60 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -58,7 +58,7 @@ $ export INGRESS_HOST=externalIP=$(kubectl -n get service get service -o jsonpath='{.spec.ports[?(@.name=="http")].port}') $ export INGRESS_HTTPS_PORT=$(kubectl -n get service -o jsonpath='{.spec.ports[?(@.name=="https")].port}') ``` -The command to determin host name may be slightly different with Amazon EKS: +The command to determine the Ingress Host may be slightly different with Amazon EKS: ``` $ export INGRESS_HOST=externalIP=$(kubectl -n get service -o jsonpath='{.status.loadBalancer.ingress[*].hostname}') ``` From c0d15b2c8637f71445de279d783c40040e4f7904 Mon Sep 17 00:00:00 2001 From: Latha Sivakumar Date: Thu, 28 Jan 2021 09:54:18 -0500 Subject: [PATCH 4/4] (#62) Pre_Install_Check: Add additional messages to report when All or Some Node(s) are not in READY state --- pre_install_report/README.md | 2 +- pre_install_report/library/pre_install_check.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index 2978c60..41b462e 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -58,7 +58,7 @@ $ export INGRESS_HOST=externalIP=$(kubectl -n get service get service -o jsonpath='{.spec.ports[?(@.name=="http")].port}') $ export INGRESS_HTTPS_PORT=$(kubectl -n get service -o jsonpath='{.spec.ports[?(@.name=="https")].port}') ``` -The command to determine the Ingress Host may be slightly different with Amazon EKS: +The command to determine the Ingress Host may be slightly different with Amazon Elastic Kubernetes Service(EKS): ``` $ export INGRESS_HOST=externalIP=$(kubectl -n get service -o jsonpath='{.status.loadBalancer.ingress[*].hostname}') ``` diff --git a/pre_install_report/library/pre_install_check.py b/pre_install_report/library/pre_install_check.py index 55ff145..1b5da78 100644 --- a/pre_install_report/library/pre_install_check.py +++ b/pre_install_report/library/pre_install_check.py @@ -739,7 +739,7 @@ def evaluate_nodes(self, nodes_data, global_data, cluster_info, quantity_): alloc_memory = str(node['allocatableMemory']) total_memory = total_memory + quantity_(str(node['memory'])) total_allocatable_memory = total_allocatable_memory + quantity_(alloc_memory) - # nodestatus = False + try: nodeReady = str(node['Ready']) if nodeReady == "True":