Merge pull request #63 from sassoftware/lasivasas_feature_144

(#62) Pre_Install_Check: Add additional messages to report when All o…

pre_install_report/README.md

@@ -58,6 +58,10 @@ $ export INGRESS_HOST=externalIP=$(kubectl -n <ingress-namespace> get service <n
 $ export INGRESS_HTTP_PORT=$(kubectl -n <ingress-namespace> get service <nginx-ingress-controller-name> -o jsonpath='{.spec.ports[?(@.name=="http")].port}')
 $ export INGRESS_HTTPS_PORT=$(kubectl -n <ingress-namespace> get service <nginx-ingress-controller-name> -o jsonpath='{.spec.ports[?(@.name=="https")].port}')
 ```
+The command to determine the Ingress Host may be slightly different with Amazon Elastic Kubernetes Service(EKS):
+```
+$ export INGRESS_HOST=externalIP=$(kubectl -n <ingress-namespace> get service <nginx-ingress-controller-name> -o jsonpath='{.status.loadBalancer.ingress[*].hostname}')
+```
 
 Use the values gathered on the command line for http or https as appropriate for your deployment:
 

pre_install_report/library/pre_install_check.py

@@ -73,6 +73,7 @@ def __init__(self, sas_logger: ViyaARKLogger, viya_kubelet_version_min, viya_min
         self._viya_min_aggregate_worker_memory: Text = viya_min_aggregate_worker_memory
         self._calculated_aggregate_allocatable_memory = None
         self._workers = 0
+        self._aggregate_nodeStatus_failures = 0
 
     def _parse_release_info(self, release_info):
         """
@@ -609,11 +610,15 @@ def _check_kubelet_errors(self, global_data, aggregate_kubelet_failures):
         return: updated global data about worker nodes retrieved
         """
         aggregate_kubelet_data = {}
-        aggregate_kubelet_data.update({'aggregate_kubelet_failures': str(aggregate_kubelet_failures)})
+        node_status_msg = ""
+        if self._aggregate_nodeStatus_failures > 0:
+            node_status_msg = " Check Node(s). All Nodes NOT in Ready Status." \
+                              + ' Issues Found: ' + str(self._aggregate_nodeStatus_failures)
+        aggregate_kubelet_data.update({'aggregate_kubelet_failures': node_status_msg})
         if aggregate_kubelet_failures > 0:
             aggregate_kubelet_data.update({'aggregate_kubelet_failures':
                                            'Check Kubelet Version on nodes.' +
-                                           ' Issues Found: ' + str(aggregate_kubelet_failures)})
+                                           ' Issues Found: ' + str(aggregate_kubelet_failures) + '.' + node_status_msg})
         global_data.append(aggregate_kubelet_data)
 
         return global_data
@@ -735,6 +740,15 @@ def evaluate_nodes(self, nodes_data, global_data, cluster_info, quantity_):
             total_memory = total_memory + quantity_(str(node['memory']))
             total_allocatable_memory = total_allocatable_memory + quantity_(alloc_memory)
 
+            try:
+                nodeReady = str(node['Ready'])
+                if nodeReady == "True":
+                    pass
+                else:
+                    self._aggregate_nodeStatus_failures += 1
+            except KeyError:
+                node['Ready'] = viya_constants.KEY_NOT_FOUND
+
             if node['worker']:
                 total_cpu_cores = total_cpu_cores + alloc_cpu_cores
                 self.logger.info("worker total_cpu_cores {}".format(str(total_cpu_cores)))

pre_install_report/library/pre_install_check_permissions.py

@@ -65,6 +65,8 @@ def __init__(self, params):
         self.ingress_data[viya_constants.INGRESS_CONTROLLER] = self.ingress_controller
         self.ingress_file = "hello-ingress.yaml"
         self._storage_class_sc: List[KubernetesResource] = None
+        self._sample_deployment = 0
+        self._sample_output = ""
 
     def _set_results_cluster_admin(self, resource_key, rc):
         """
@@ -73,7 +75,9 @@ def _set_results_cluster_admin(self, resource_key, rc):
         """
         if rc == 1:
             self.cluster_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS
-            self.cluster_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] = viya_constants.INSUFFICIENT_PERMS
+            self.cluster_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] = \
+                viya_constants.INSUFFICIENT_PERMS + ". Check Logs."
+
         else:
             self.cluster_admin_permission_data[resource_key] = viya_constants.ADEQUATE_PERMS
 
@@ -82,12 +86,41 @@ def _set_results_namespace_admin(self, resource_key, rc):
         Set permissions status for specified resource/verb with namespace admin role
 
         """
-        if rc == 1:
-            self.namespace_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS
-            self.namespace_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] \
-                = viya_constants.INSUFFICIENT_PERMS
+        sample_keys = [viya_constants.PERM_DEPLOYMENT]
+        deployment_keys = [viya_constants.PERM_DELETE + viya_constants.PERM_DEPLOYMENT,
+                           viya_constants.PERM_SERVICE,
+                           viya_constants.PERM_DELETE + viya_constants.PERM_SERVICE,
+                           viya_constants.PERM_INGRESS,
+                           viya_constants.PERM_DELETE + viya_constants.PERM_INGRESS,
+                           viya_constants.PERM_REPLICASET,
+                           viya_constants.PERM_CREATE + viya_constants.PERM_ROLE,
+                           viya_constants.PERM_CREATE + viya_constants.PERM_ROLEBINDING,
+                           viya_constants.PERM_CREATE + viya_constants.PERM_SA,
+                           viya_constants.PERM_DELETE + viya_constants.PERM_ROLE,
+                           viya_constants.PERM_DELETE + viya_constants.PERM_ROLEBINDING,
+                           viya_constants.PERM_DELETE + viya_constants.PERM_SA
+                           ]
+        if rc != 0:
+            self.logger.debug("resource_key = {}, sample_deployment = {} ".format(str(resource_key),
+                                                                                  str(self._sample_deployment)))
+            if self._sample_deployment != 0:
+                if resource_key in deployment_keys:
+                    self.namespace_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS
+                if resource_key in sample_keys:
+                    self.namespace_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS + \
+                                                                         ". Sample Deployment Check failed! " + \
+                                                                         "Ensure Node(s) Status is Ready. " + \
+                                                                         "Check Permissions in specified namespace. " \
+                                                                         + self._sample_output
+
+            else:
+                self.namespace_admin_permission_data[resource_key] = viya_constants.INSUFFICIENT_PERMS
+            self.namespace_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] = \
+                viya_constants.INSUFFICIENT_PERMS + ". Check Logs."
         else:
             self.namespace_admin_permission_data[resource_key] = viya_constants.ADEQUATE_PERMS
+            # self.namespace_admin_permission_aggregate[viya_constants.PERM_PERMISSIONS] = \
+            #    viya_constants.ADEQUATE_PERMS
 
     def _get_pvc(self, pvc_name, key):
         """
@@ -284,20 +317,34 @@ def check_sample_application(self):
 
         rc = self.utils.deploy_manifest_file(viya_constants.KUBECTL_APPLY,
                                              'hello-application.yaml')
-        #    self._set_results_namespace_admin(viya_constants.PERM_DEPLOYMENT, rc)
-        #    self._set_results_namespace_admin(viya_constants.PERM_SERVICE, rc)
 
         if rc == 0:
-            rc = self.utils.do_cmd(" rollout status deployment.v1.apps/hello-world ")
+            rc, sample_output = self.utils.do_cmd(" rollout status deployment.v1.apps/hello-world --timeout=180s")
+            # You can check if a Deployment has completed by using kubectl rollout status.
+            # If the rollout completed successfully, kubectl rollout status returns a zero exit code.
+
+            if rc != 0:
+                self._sample_deployment = 2
+                self._sample_output = sample_output
+                self._set_results_namespace_admin(viya_constants.PERM_DEPLOYMENT, rc)
+                self._set_results_namespace_admin(viya_constants.PERM_SERVICE, rc)
+                return 2
 
             self._set_results_namespace_admin(viya_constants.PERM_DEPLOYMENT, rc)
             self._set_results_namespace_admin(viya_constants.PERM_SERVICE, rc)
 
             if rc == 0:
-                rc = self.utils.do_cmd(" scale --replicas=2 deployment/hello-world ")
+                rc, sample_output = self.utils.do_cmd(" scale --replicas=2 deployment/hello-world ")
+                if rc != 0:
+                    self._sample_deployment = 3
+                self._set_results_namespace_admin(viya_constants.PERM_REPLICASET, rc)
+                return 3
+        else:
+            self._sample_deployment = 1
+            self._set_results_namespace_admin(viya_constants.PERM_DEPLOYMENT, rc)
+            self._set_results_namespace_admin(viya_constants.PERM_SERVICE, rc)
 
-                if rc == 0:
-                    self._set_results_namespace_admin(viya_constants.PERM_REPLICASET, rc)
+            return 1
 
     def check_sample_service(self):
         """
@@ -396,8 +443,9 @@ def check_delete_sample_application(self):
         rc = self.utils.deploy_manifest_file(viya_constants.KUBECTL_DELETE,
                                              'hello-application.yaml')
         self._set_results_namespace_admin(viya_constants.PERM_DELETE + viya_constants.PERM_DEPLOYMENT, rc)
+        self._set_results_namespace_admin(viya_constants.PERM_DELETE + viya_constants.PERM_SERVICE, rc)
 
-        rc = self.utils.do_cmd(" wait --for=delete pod -l app=hello-world-pod --timeout=12s ")
+        self.utils.do_cmd(" wait --for=delete pod -l app=hello-world-pod --timeout=12s ")
 
     def check_delete_sample_service(self):
         """
@@ -443,12 +491,13 @@ def check_deploy_crd(self):
     def check_rbac_role(self):
         """
         Check if RBAC is enabled in specified namespace
-        Create the Role and Rolebinding for the custome resource access with specified namespace. Set the
+        Create the Role and Rolebinding for the custom resource access with specified namespace. Set the
         permissions status in the namespace_admin_permission_data dict object.
 
         """
         found = self.utils.get_rbac_group_cmd()
-
+        self.logger.debug("get_rbace_group_cmd found = {}, sample_deployment = {}"
+                          .format(str(found), str(self._sample_deployment)))
         if found:
             rc = self.utils.deploy_manifest_file(viya_constants.KUBECTL_APPLY,
                                                  'viya-role.yaml')
@@ -463,7 +512,14 @@ def check_rbac_role(self):
                                                  'viya-rolebinding.yaml')
             self._set_results_namespace_admin(viya_constants.PERM_CREATE + viya_constants.PERM_ROLEBINDING, rc)
         else:
+            self.logger.debug("sample_deployment = {}".format(str(self._sample_deployment)))
             self.namespace_admin_permission_aggregate["RBAC Checking"] = viya_constants.PERM_SKIPPING
+            self._set_results_namespace_admin(viya_constants.PERM_CREATE + viya_constants.PERM_ROLE,
+                                              int(self._sample_deployment))
+            self._set_results_namespace_admin(viya_constants.PERM_CREATE + viya_constants.PERM_SA,
+                                              int(self._sample_deployment))
+            self._set_results_namespace_admin(viya_constants.PERM_CREATE + viya_constants.PERM_ROLEBINDING,
+                                              int(self._sample_deployment))
 
     def check_rbac_delete_role(self):
         """
@@ -495,14 +551,14 @@ def check_get_custom_resource(self, namespace):
         if not allowed:
             rc1 = 1
 
-        self._set_results_namespace_admin_crd(viya_constants.PERM_CREATE + viya_constants.PERM_CR + " with RBAC "
-                                              + viya_constants.PERM_SA + " resp: = " + str(allowed), rc1)
+        self._set_results_namespace_admin_crd(viya_constants.PERM_CREATE + viya_constants.PERM_CR_RBAC
+                                              + viya_constants.PERM_SA, rc1)
         allowed: bool = self.utils.can_i(' delete viyas.company.com --as=system:serviceaccount:'
                                          + namespace + ':crreader ')
         if allowed:
             rc2 = 1
-        self._set_results_namespace_admin_crd(viya_constants.PERM_DELETE + viya_constants.PERM_CR + " with RBAC "
-                                              + viya_constants.PERM_SA + " resp: = " + str(allowed), rc2)
+        self._set_results_namespace_admin_crd(viya_constants.PERM_DELETE + viya_constants.PERM_CR_RBAC
+                                              + viya_constants.PERM_SA, rc2)
 
     def check_delete_custom_resource(self):
         """

pre_install_report/library/pre_install_utils.py

@@ -77,12 +77,12 @@ def do_cmd(self, test_cmd):
 
             self.logger.info("cmd {} rc = 0".format(test_cmd))
             self.logger.debug("cmd {} rc = 0 response {}".format(test_cmd, str(data)))
-            return 0
+            return 0, str(data)
         except CalledProcessError as e:
             data = e.output
             self.logger.error("do_cmd " + ' rc = ' + str(e.returncode) + test_cmd +
                               ' data = ' + str(data))
-            return e.returncode
+            return e.returncode, str(data)
 
     def get_rbac_group_cmd(self):
         """
@@ -91,14 +91,14 @@ def get_rbac_group_cmd(self):
         cmd: kubectl command to retrieve api_resources
         return:  True if both Role and RoleBinding kinds have an api_group
         """
-        role = None
-        rolebinding = None
+        role: bool = None
+        rolebinding: bool = None
         try:
             data: KubernetesApiResources = self._kubectl.api_resources(False)
             role = data.get_api_group("Role")
             rolebinding = data.get_api_group("RoleBinding")
         except CalledProcessError as e:
-            self.logger.exception("get_rbac_group_cmd  rc {}" + str(e.returncode))
+            self.logger.exception("get_rbac_group_cmd  rc {} ".format(str(e.returncode)))
             return False
         if role is None:
             return False

pre_install_report/library/utils/viya_constants.py

@@ -50,3 +50,4 @@
 PERM_ROLEBINDING = "RoleBinding"
 PERM_SA = "Service Account"
 PERM_CLASS = "PreInstallUtils"
+PERM_CR_RBAC = "Custom Resource with RBAC "

pre_install_report/templates/report_template_viya_pre_install_check.j2

@@ -189,6 +189,7 @@
                      <TH> Kubelet Version </TH>
                      <TH> Container Runtime</TH>
                      <TH> Kernel Version </TH>
+                     <TH> Node Status </TH>
                      <TH> Issues </TH>
                    </TR>
                    {% for node in nodes_data %}
@@ -197,6 +198,7 @@
                      <TD>{{node.kubeletversion}}</TD>
                      <TD>{{node.containerRuntimeVersion}}</TD>
                      <TD>{{node.kernelVersion}}</TD>
+                     <TD>{{node.Ready}}</TD>
                      <TD>{{node.error.kubeletversion}}</TD>
                    </TR>
                    {% endfor %}
@@ -355,7 +357,7 @@
 
 <!-- NameSpace Admin Permissions -->
     <div class="jq-accordion">
-        <h2>Namespace Admin Permissions: {{ namespace_admin_permission_aggregate }} </h2>
+        <h2>Namespace Admin Permissions - {{ namespace_admin_permission_aggregate }} </h2>
     <div>
 
     <table class="mytable">

pre_install_report/test/test_data/json_data/azure_nodes_no_master.json

@@ -112,7 +112,7 @@
                         "lastTransitionTime": "2020-09-02T20:10:38Z",
                         "message": "kubelet is posting ready status. AppArmor enabled",
                         "reason": "KubeletReady",
-                        "status": "True",
+                        "status": "Unknown",
                         "type": "Ready"
                     }
                 ],
@@ -651,7 +651,7 @@
                         "lastTransitionTime": "2020-09-02T20:10:08Z",
                         "message": "kubelet is posting ready status. AppArmor enabled",
                         "reason": "KubeletReady",
-                        "status": "True",
+                        "status": "Unknown",
                         "type": "Ready"
                     }
                 ],
@@ -816,7 +816,7 @@
                         "lastTransitionTime": "2020-09-02T20:09:34Z",
                         "message": "kubelet is posting ready status. AppArmor enabled",
                         "reason": "KubeletReady",
-                        "status": "True",
+                        "status": "Unknown",
                         "type": "Ready"
                     }
                 ],
@@ -993,7 +993,7 @@
                         "lastTransitionTime": "2020-09-02T20:09:38Z",
                         "message": "kubelet is posting ready status. AppArmor enabled",
                         "reason": "KubeletReady",
-                        "status": "True",
+                        "status": "Unknown",
                         "type": "Ready"
                     }
                 ],
@@ -1175,7 +1175,7 @@
                         "lastTransitionTime": "2020-09-02T20:09:54Z",
                         "message": "kubelet is posting ready status. AppArmor enabled",
                         "reason": "KubeletReady",
-                        "status": "True",
+                        "status": "Unknown",
                         "type": "Ready"
                     }
                 ],
@@ -1326,7 +1326,7 @@
                         "lastTransitionTime": "2020-09-02T20:09:42Z",
                         "message": "kubelet is posting ready status. AppArmor enabled",
                         "reason": "KubeletReady",
-                        "status": "True",
+                        "status": "Unknown",
                         "type": "Ready"
                     }
                 ],
@@ -1469,7 +1469,7 @@
                         "lastTransitionTime": "2020-09-02T20:06:29Z",
                         "message": "kubelet is posting ready status. AppArmor enabled",
                         "reason": "KubeletReady",
-                        "status": "True",
+                        "status": "Unknown",
                         "type": "Ready"
                     }
                 ],
@@ -1612,7 +1612,7 @@
                         "lastTransitionTime": "2020-09-02T20:06:46Z",
                         "message": "kubelet is posting ready status. AppArmor enabled",
                         "reason": "KubeletReady",
-                        "status": "True",
+                        "status": "Unknown",
                         "type": "Ready"
                     }
                 ],