Merge pull request #186 from sassoftware/develop

Prepare for release 1.9.0

.github/workflows/ci-validation.yml

@@ -23,11 +23,13 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install flake8 pytest
+        flake8 --version
+        pytest --version
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --max-line-length 120 --show-source --statistics
+        flake8 . --count --max-line-length 120 --show-source --statistics --extend-ignore=E275
     - name: Test with pytest
       run: |
         pytest

deployment_report/deployment_report.py

@@ -80,6 +80,10 @@ def main(argv: List):
     arg_parser.add_argument(
         "-d", "--data-file-only", action="store_true", dest="data_file_only",
         help="Generate only the JSON-formatted data.")
+    # ingress namespace
+    arg_parser.add_argument(
+        "-i", "--ingress-namespace", type=Text, default=None, dest="ingress_namespace",
+        help="Ingress namespace in the target cluster.")
     # kubectl-global-opts
     arg_parser.add_argument(
         "-k", "--kubectl-global-opts", type=Text, default="", dest="kubectl_global_opts",
@@ -111,7 +115,8 @@ def main(argv: List):
     # initialize the kubectl object
     # this will also verify the connection to the cluster and if the namespace is valid, if provided
     try:
-        kubectl: Kubectl = Kubectl(namespace=args.namespace, global_opts=args.kubectl_global_opts)
+        kubectl: Kubectl = Kubectl(namespace=args.namespace, global_opts=args.kubectl_global_opts,
+                                   ingress_namespace=args.ingress_namespace)
     except ConnectionError as e:
         print()
         print(f"ERROR: {e}", file=sys.stderr)

deployment_report/model/static/viya_deployment_report_keys.py

@@ -49,6 +49,7 @@ class Kubernetes(object):
         DB_INFO = "dbInfo"
         DISCOVERED_RESOURCE_TYPES_DICT = "discoveredResourceTypes"
         INGRESS_CTRL = "ingressController"
+        INGRESS_VER = "ingressVersion"
         NAMESPACE = "namespace"
         NODES_DICT = "nodes"
         SECRETS_DICT = "secrets"

deployment_report/model/test/test_viya_deployment_report.py

@@ -89,14 +89,15 @@ def test_get_kubernetes_details(report: ViyaDeploymentReport) -> None:
     kube_details: Dict = report.get_kubernetes_details()
 
     # check for all expected entries
-    assert len(kube_details) == 11
+    assert len(kube_details) == 12
     assert ReportKeys.Kubernetes.API_RESOURCES_DICT in kube_details
     assert ReportKeys.Kubernetes.API_VERSIONS_LIST in kube_details
     assert ReportKeys.Kubernetes.CADENCE_INFO in kube_details
     assert ReportKeys.Kubernetes.CONFIGMAPS_DICT in kube_details
     assert ReportKeys.Kubernetes.DB_INFO in kube_details
     assert ReportKeys.Kubernetes.DISCOVERED_RESOURCE_TYPES_DICT in kube_details
     assert ReportKeys.Kubernetes.INGRESS_CTRL in kube_details
+    assert ReportKeys.Kubernetes.INGRESS_VER in kube_details
     assert ReportKeys.Kubernetes.NAMESPACE in kube_details
     assert ReportKeys.Kubernetes.NODES_DICT in kube_details
     assert ReportKeys.Kubernetes.SECRETS_DICT in kube_details

deployment_report/model/utils/config_util.py

@@ -4,7 +4,7 @@
 # ### Author: SAS Institute Inc.                                 ###
 ####################################################################
 #                                                                ###
-# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA.         ###
+# Copyright (c) 2022, SAS Institute Inc., Cary, NC, USA.         ###
 # All Rights Reserved.                                           ###
 # SPDX-License-Identifier: Apache-2.0                            ###
 #                                                                ###
@@ -47,6 +47,7 @@
 _UNSUPPORTED_DB_ = "Unsupported"
 _CRUNCH_DB_ = "crunch"
 _CRUNCHDATA_DB_ = "sas-crunchy-data-"
+_CRUNCH5DATA_DB_ = "sas-crunchy-platform-"
 _ALT_DB_ = "sas.database.alternative.databaseServerName"
 _UNAVAIL_DB_ = "not available"
 
@@ -116,6 +117,10 @@ def get_db_info(resource_cache: Dict) -> Dict:
     if not resource_cache:
         return None
 
+    if ResourceTypeValues.SAS_CRUNCHYCLUSTERS in resource_cache.keys():
+        pgclusters: Dict = resource_cache[ResourceTypeValues.SAS_CRUNCHYCLUSTERS][ITEMS_KEY]
+        db_dict = _get_db_info_v3(pgclusters)
+
     if ResourceTypeValues.SAS_PGCLUSTERS in resource_cache.keys():
         pgclusters: Dict = resource_cache[ResourceTypeValues.SAS_PGCLUSTERS][ITEMS_KEY]
         db_dict = _get_db_info_v2(pgclusters)
@@ -188,6 +193,51 @@ def _get_db_info_v1(config_maps: Dict) -> Dict:
     return db_dict
 
 
+def _get_db_info_v3(pgclusters: Dict) -> Dict:
+    """
+    Returns the db information of the targeted SAS deployment.
+
+    :param pgclusters: The pgclusters resource to evaluate.
+    :return: A dictionary representing the db information of the targeted SAS deployment.
+    """
+    # initialize the return value
+    db_dict: Dict = dict()
+    for key in pgclusters:
+        try:
+            resource_definition = pgclusters[key][Keys.ResourceDetails.RESOURCE_DEFINITION]
+
+            db_name: Optional[Text] = resource_definition.get_name()
+            db_data: Optional[Dict] = resource_definition.get_spec()
+
+            if not db_data:
+                continue
+
+            dbs: Dict = dict()
+
+            try:
+                if (db_data['port']):
+                    dbs = {Keys.DatabaseDetails.DBTYPE: _INTERNAL_DB_}
+                else:
+                    dbs = {
+                        Keys.DatabaseDetails.DBTYPE: _EXTERNAL_DB_,
+                        Keys.DatabaseDetails.DBCONN: _UNAVAIL_DB_
+                    }
+            except KeyError:
+                continue
+
+            if dbs:
+                # convert db_name to be aligned
+                if _CRUNCH5DATA_DB_ in db_name:
+                    db_name = db_name.replace(_CRUNCH5DATA_DB_, "")
+
+                db_dict[db_name] = dbs
+
+        except KeyError:
+            continue
+
+    return db_dict
+
+
 def _get_db_info_v2(pgclusters: Dict) -> Dict:
     """
     Returns the db information of the targeted SAS deployment.

deployment_report/model/utils/ingress_util.py

@@ -9,7 +9,7 @@
 # SPDX-License-Identifier: Apache-2.0                            ###
 #                                                                ###
 ####################################################################
-from typing import Dict, Optional, Text
+from typing import AnyStr, Dict, List, Optional, Text
 
 from deployment_report.model.static.viya_deployment_report_keys import \
     ITEMS_KEY, \
@@ -18,6 +18,12 @@
 from viya_ark_library.k8s.k8s_resource_type_values import KubernetesResourceTypeValues as ResourceTypeValues
 from viya_ark_library.k8s.sas_k8s_ingress import SupportedIngress
 from viya_ark_library.k8s.sas_k8s_objects import KubernetesResource
+from viya_ark_library.k8s.sas_kubectl_interface import KubectlInterface
+
+
+# constants values
+_NGINX_VERSION_ = "nginx version:"
+_RELEASE_ = "Release:"
 
 
 def determine_ingress_controller(gathered_resources: Dict) -> Optional[Text]:
@@ -103,3 +109,62 @@ def ignorable_for_controller_if_unavailable(ingress_controller: Text, resource_t
 
     # not ignorable
     return False
+
+
+def get_ingress_version(kubectl: KubectlInterface) -> Optional[Text]:
+    """
+    Retrieves ingress version used in the Kubernetes cluster
+
+    :param kubectl: The KubectlInterface object.
+    :return: The ingress controller version used in the target cluster or Blank if it cannot be determined.
+    """
+
+    version: Text = ""
+    getpod_cmd: AnyStr = "get pods -n " + kubectl.ingress_ns + \
+                         " --field-selector=status.phase==Running" + \
+                         " -o jsonpath=\"{.items[0].metadata.name}\""
+
+    if kubectl.ingress_ns == SupportedIngress.Controllers.NS_NGINX:
+        podname: AnyStr = kubectl.do(getpod_cmd +
+                                     " -l app.kubernetes.io/component=controller")
+        version_str: AnyStr = kubectl.do("exec -it " + podname.decode() +
+                                         " -n " + kubectl.ingress_ns +
+                                         " -- /nginx-ingress-controller --version")
+        version_list: List = version_str.decode().splitlines()
+        for v in version_list:
+            if _RELEASE_ in v:
+                version = ' '.join(v.split()) + version
+            elif _NGINX_VERSION_ in v:
+                version = version + ", " + v.split()[-1]
+
+    elif kubectl.ingress_ns == SupportedIngress.Controllers.NS_ISTIO:
+        podname: AnyStr = kubectl.do(getpod_cmd +
+                                     " -l  app=istiod")
+        version_str: AnyStr = kubectl.do("exec -it " + podname.decode() +
+                                         " -n " + kubectl.ingress_ns +
+                                         " -- pilot-discovery version --short")
+        version = version_str.decode()
+
+    elif kubectl.ingress_ns == SupportedIngress.Controllers.NS_OPENSHIFT:
+        podname: AnyStr = kubectl.do(getpod_cmd +
+                                     " -l  name=ingress-operator")
+        version_str: AnyStr = kubectl.do("get pod " + podname.decode() +
+                                         " -n " + kubectl.ingress_ns +
+                                         " -o jsonpath=\"{.spec.containers[].env[" +
+                                         "?(@.name=='RELEASE_VERSION')].value}\"")
+        version = version_str.decode()
+
+    elif kubectl.ingress_ns == SupportedIngress.Controllers.NS_CONTOUR:
+        podname: AnyStr = kubectl.do(getpod_cmd +
+                                     " -l app=envoy")
+        version_str: AnyStr = kubectl.do("get pod " + podname.decode() +
+                                         " -n " + kubectl.ingress_ns +
+                                         " -o jsonpath=\"{.spec.containers[*].image}\"")
+        version_list: List = version_str.decode().split(' ')
+        for v in version_list:
+            if version:
+                version = version + ", " + v.split("/")[-1].capitalize()
+            else:
+                version = v.split("/")[-1].capitalize()
+
+    return version.strip()

deployment_report/model/utils/test/test_config_util.py

@@ -4,7 +4,7 @@
 # ### Author: SAS Institute Inc.                                 ###
 ####################################################################
 #                                                                ###
-# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA.         ###
+# Copyright (c) 2022, SAS Institute Inc., Cary, NC, USA.         ###
 # All Rights Reserved.                                           ###
 # SPDX-License-Identifier: Apache-2.0                            ###
 #                                                                ###
@@ -62,6 +62,20 @@ def test_get_db_info_v2(no_ingress_simulation_fixture: conftest.DSA):
     assert db_dict[config_util._DBNAME_POSTGRES_][Keys.DatabaseDetails.DBTYPE] == KubectlTest.Values.DB_External
 
 
+@pytest.mark.usefixtures(conftest.NO_INGRESS_SIMULATION_FIXTURE)
+def test_get_db_info_v3(no_ingress_simulation_fixture: conftest.DSA):
+    """
+    This test verifies that the provided db data is returned
+    """
+    # get the resource cache
+    resource_cache: Dict = no_ingress_simulation_fixture.resource_cache()
+
+    pgclusters: Dict = resource_cache[ResourceTypeValues.SAS_CRUNCHYCLUSTERS][ITEMS_KEY]
+    db_dict: Dict = config_util._get_db_info_v3(pgclusters=pgclusters)
+
+    assert db_dict[config_util._DBNAME_POSTGRES_][Keys.DatabaseDetails.DBTYPE] == KubectlTest.Values.DB_Internal
+
+
 @pytest.mark.usefixtures(conftest.NO_INGRESS_SIMULATION_FIXTURE)
 def test_get_db_info_v1(no_ingress_simulation_fixture: conftest.DSA):
     """

deployment_report/model/viya_deployment_report.py

@@ -277,13 +277,21 @@ def gather_details(self, kubectl: KubectlInterface,
         #######################################################################
         # default values in-case pods were not found
         ingress_controller: Optional[Text] = None
+        ingress_version: Optional[Text] = None
         unavailable_resources: List = list()
 
         # evaluate resources that would be gathered if pods were found
         if pods_found:
             # determine the ingress controller for the deployment
             # this will help to evaluate which resources should be considered "unavailable"
             ingress_controller = ingress_util.determine_ingress_controller(resource_cache)
+            if (
+                ingress_controller == SupportedIngress.Controllers.NGINX or
+                ingress_controller == SupportedIngress.Controllers.ISTIO or
+                ingress_controller == SupportedIngress.Controllers.OPENSHIFT or
+                ingress_controller == SupportedIngress.Controllers.CONTOUR
+               ):
+                ingress_version = ingress_util.get_ingress_version(kubectl)
 
             # determine if any resource types for which caching was attempted were unavailable
             # if at least one is unavailable, a message will be displayed saying that components may not be complete
@@ -361,6 +369,9 @@ def gather_details(self, kubectl: KubectlInterface,
         # create a key to mark the determined ingress controller for the cluster: str|None
         k8s_details_dict[Keys.Kubernetes.INGRESS_CTRL]: Optional[Text] = ingress_controller
 
+        # create a key to mark the determined ingress version for the cluster: str|None
+        k8s_details_dict[Keys.Kubernetes.INGRESS_VER]: Optional[Text] = ingress_version
+
         # create a key to mark the namespace evaluated for this report: str|None
         k8s_details_dict[Keys.Kubernetes.NAMESPACE] = kubectl.get_namespace()
 

deployment_report/templates/viya_deployment_report.html.j2

@@ -89,10 +89,6 @@
       <th>Namespace</th>
       <td>{{ report_data.kubernetes.namespace | default("<em>could not be determined</em>") }}</td>
     </tr>
-    <tr>
-      <th>Ingress Controller</th>
-      <td>{{ report_data.kubernetes.ingressController | default("<em>could not be determined</em>") }}</td>
-    </tr>
     <tr>
       <th>Cadence Version</th>
       <td>{{ report_data.kubernetes.cadenceInfo | default("<em>could not be determined</em>") }}</td>
@@ -104,6 +100,7 @@
   <div class="jq-accordion">
     <h2>Versions</h2>
     <div>
+      <h7>Kubernetes</h7>
       <table>
         <tr>
           <th></th>
@@ -126,6 +123,21 @@
           <td>{{ report_data.kubernetes.versions.serverVersion.platform }}</td>
         </tr>
       </table>
+      <h7>Ingress</h7>
+      <table>
+        <tr>
+          <th>Controller</th>
+          <td>{{ report_data.kubernetes.ingressController | default("<em>could not be determined</em>") }}</td>
+        </tr>
+        <tr>
+          <th>Version</th>
+          {% if report_data.kubernetes.ingressVersion is not none and report_data.kubernetes.ingressVersion | length > 0 %}
+            <td>{{ report_data.kubernetes.ingressVersion}}</td>
+          {% else %}
+            <td>Unavailable</td>
+          {% endif %}
+        </tr>
+      </table>
     </div>
   </div>
   {# Cluster Overview: Kubernetes Versions Accordion #}

deployment_report/test/data/expected_usage_output.txt

@@ -1,4 +1,5 @@
-usage: viya-ark.py deployment-report [-h] [-d] [-k KUBECTL_GLOBAL_OPTS] [-l]
+usage: viya-ark.py deployment-report [-h] [-d] [-i INGRESS_NAMESPACE]
+                                     [-k KUBECTL_GLOBAL_OPTS] [-l]
                                      [-n NAMESPACE] [-o OUTPUT_DIR] [-r]
 
 Generate a deployment report of SAS components for a target Kubernetes
@@ -7,6 +8,8 @@ environment.
 optional arguments:
   -h, --help            show this help message and exit
   -d, --data-file-only  Generate only the JSON-formatted data.
+  -i INGRESS_NAMESPACE, --ingress-namespace INGRESS_NAMESPACE
+                        Ingress namespace in the target cluster.
   -k KUBECTL_GLOBAL_OPTS, --kubectl-global-opts KUBECTL_GLOBAL_OPTS
                         Any kubectl global options to use with all executions
                         (excluding namespace, which should be set using -n,