From 01ad832dcc65b6a92632df5d07495ae01e33742b Mon Sep 17 00:00:00 2001 From: Amy Ho <39809594+cuddlehub@users.noreply.github.com> Date: Fri, 1 Jul 2022 10:13:33 -0400 Subject: [PATCH 01/20] Issue 123 (#174) * (#123) Request to include version information for ingress controller * (#123) Request to include version information for ingress controller * (#123) Request to include version information for ingress controller Co-authored-by: Kevin Lingle <36995745+kevinlinglesas@users.noreply.github.com> Co-authored-by: lasiva <45047672+lasiva@users.noreply.github.com> --- deployment_report/deployment_report.py | 7 +- .../static/viya_deployment_report_keys.py | 1 + .../model/test/test_viya_deployment_report.py | 3 +- deployment_report/model/utils/ingress_util.py | 70 ++++++++++++++++++- .../model/viya_deployment_report.py | 11 +++ .../templates/viya_deployment_report.html.j2 | 20 ++++-- .../test/data/expected_usage_output.txt | 5 +- viya_ark_library/k8s/sas_k8s_ingress.py | 4 ++ viya_ark_library/k8s/sas_kubectl.py | 41 +++++++++-- .../k8s/test_impl/sas_kubectl_test.py | 8 ++- 10 files changed, 156 insertions(+), 14 deletions(-) diff --git a/deployment_report/deployment_report.py b/deployment_report/deployment_report.py index 65ec483..7f96ebd 100644 --- a/deployment_report/deployment_report.py +++ b/deployment_report/deployment_report.py @@ -80,6 +80,10 @@ def main(argv: List): arg_parser.add_argument( "-d", "--data-file-only", action="store_true", dest="data_file_only", help="Generate only the JSON-formatted data.") + # ingress namespace + arg_parser.add_argument( + "-i", "--ingress-namespace", type=Text, default=None, dest="ingress_namespace", + help="Ingress namespace in the target cluster.") # kubectl-global-opts arg_parser.add_argument( "-k", "--kubectl-global-opts", type=Text, default="", dest="kubectl_global_opts", @@ -111,7 +115,8 @@ def main(argv: List): # initialize the kubectl object # this will also verify the connection to the cluster and if the namespace is valid, if provided try: - kubectl: Kubectl = Kubectl(namespace=args.namespace, global_opts=args.kubectl_global_opts) + kubectl: Kubectl = Kubectl(namespace=args.namespace, global_opts=args.kubectl_global_opts, + ingress_namespace=args.ingress_namespace) except ConnectionError as e: print() print(f"ERROR: {e}", file=sys.stderr) diff --git a/deployment_report/model/static/viya_deployment_report_keys.py b/deployment_report/model/static/viya_deployment_report_keys.py index af24efb..9d7b5cf 100644 --- a/deployment_report/model/static/viya_deployment_report_keys.py +++ b/deployment_report/model/static/viya_deployment_report_keys.py @@ -49,6 +49,7 @@ class Kubernetes(object): DB_INFO = "dbInfo" DISCOVERED_RESOURCE_TYPES_DICT = "discoveredResourceTypes" INGRESS_CTRL = "ingressController" + INGRESS_VER = "ingressVersion" NAMESPACE = "namespace" NODES_DICT = "nodes" SECRETS_DICT = "secrets" diff --git a/deployment_report/model/test/test_viya_deployment_report.py b/deployment_report/model/test/test_viya_deployment_report.py index ac973f4..ec28d22 100644 --- a/deployment_report/model/test/test_viya_deployment_report.py +++ b/deployment_report/model/test/test_viya_deployment_report.py @@ -89,7 +89,7 @@ def test_get_kubernetes_details(report: ViyaDeploymentReport) -> None: kube_details: Dict = report.get_kubernetes_details() # check for all expected entries - assert len(kube_details) == 11 + assert len(kube_details) == 12 assert ReportKeys.Kubernetes.API_RESOURCES_DICT in kube_details assert ReportKeys.Kubernetes.API_VERSIONS_LIST in kube_details assert ReportKeys.Kubernetes.CADENCE_INFO in kube_details @@ -97,6 +97,7 @@ def test_get_kubernetes_details(report: ViyaDeploymentReport) -> None: assert ReportKeys.Kubernetes.DB_INFO in kube_details assert ReportKeys.Kubernetes.DISCOVERED_RESOURCE_TYPES_DICT in kube_details assert ReportKeys.Kubernetes.INGRESS_CTRL in kube_details + assert ReportKeys.Kubernetes.INGRESS_VER in kube_details assert ReportKeys.Kubernetes.NAMESPACE in kube_details assert ReportKeys.Kubernetes.NODES_DICT in kube_details assert ReportKeys.Kubernetes.SECRETS_DICT in kube_details diff --git a/deployment_report/model/utils/ingress_util.py b/deployment_report/model/utils/ingress_util.py index 03c9b4a..7004099 100644 --- a/deployment_report/model/utils/ingress_util.py +++ b/deployment_report/model/utils/ingress_util.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: Apache-2.0 ### # ### #################################################################### -from typing import Dict, Optional, Text +from typing import AnyStr, Dict, List, Optional, Text from deployment_report.model.static.viya_deployment_report_keys import \ ITEMS_KEY, \ @@ -18,6 +18,12 @@ from viya_ark_library.k8s.k8s_resource_type_values import KubernetesResourceTypeValues as ResourceTypeValues from viya_ark_library.k8s.sas_k8s_ingress import SupportedIngress from viya_ark_library.k8s.sas_k8s_objects import KubernetesResource +from viya_ark_library.k8s.sas_kubectl_interface import KubectlInterface + + +# constants values +_NGINX_VERSION_ = "nginx version:" +_RELEASE_ = "Release:" def determine_ingress_controller(gathered_resources: Dict) -> Optional[Text]: @@ -103,3 +109,65 @@ def ignorable_for_controller_if_unavailable(ingress_controller: Text, resource_t # not ignorable return False + + +def get_ingress_version(kubectl: KubectlInterface) -> Optional[Text]: + """ + Retrieves ingress version used in the Kubernetes cluster + + :param kubectl: The KubectlInterface object. + :return: The ingress controller version used in the target cluster or Blank if it cannot be determined. + """ + + version: Text = "" + + if kubectl.ingress_ns == SupportedIngress.Controllers.NS_NGINX: + podname: AnyStr = kubectl.do("get pods -n " + + kubectl.ingress_ns + + " -l app.kubernetes.io/component=controller" + + " -o jsonpath='{.items[0].metadata.name}'") + version_str: AnyStr = kubectl.do("exec -it " + podname.decode() + + " -n " + kubectl.ingress_ns + + " -- /nginx-ingress-controller --version") + version_list: List = version_str.decode().splitlines() + for v in version_list: + if _RELEASE_ in v: + version = ' '.join(v.split()) + version + elif _NGINX_VERSION_ in v: + version = version + ", " + v.split()[-1] + + elif kubectl.ingress_ns == SupportedIngress.Controllers.NS_ISTIO: + podname: AnyStr = kubectl.do("get pods -n " + kubectl.ingress_ns + + " -l app=istiod" + + " -o jsonpath='{.items[0].metadata.name}'") + version_str: AnyStr = kubectl.do("exec -it " + podname.decode() + + " -n " + kubectl.ingress_ns + + " -- pilot-discovery version --short") + version = version_str.decode() + + elif kubectl.ingress_ns == SupportedIngress.Controllers.NS_OPENSHIFT: + podname: AnyStr = kubectl.do("get pods -n " + kubectl.ingress_ns + + " -l name=ingress-operator" + + " -o jsonpath='{.items[0].metadata.name}'") + version_str: AnyStr = kubectl.do("get pod " + podname.decode() + + " -n " + kubectl.ingress_ns + + " -o jsonpath=\"{.spec.containers[].env[" + + "?(@.name=='RELEASE_VERSION')].value}\"") + version = version_str.decode() + + elif kubectl.ingress_ns == SupportedIngress.Controllers.NS_CONTOUR: + podname: AnyStr = kubectl.do("get pods -n " + + kubectl.ingress_ns + + " -l app=envoy" + + " -o jsonpath='{.items[0].metadata.name}'") + version_str: AnyStr = kubectl.do("get pod " + podname.decode() + + " -n " + kubectl.ingress_ns + + " -o jsonpath='{.spec.containers[*].image}'") + version_list: List = version_str.decode().split(' ') + for v in version_list: + if version: + version = version + ", " + v.split("/")[-1].capitalize() + else: + version = v.split("/")[-1].capitalize() + + return version.strip() diff --git a/deployment_report/model/viya_deployment_report.py b/deployment_report/model/viya_deployment_report.py index 7793820..2a29977 100644 --- a/deployment_report/model/viya_deployment_report.py +++ b/deployment_report/model/viya_deployment_report.py @@ -277,6 +277,7 @@ def gather_details(self, kubectl: KubectlInterface, ####################################################################### # default values in-case pods were not found ingress_controller: Optional[Text] = None + ingress_version: Optional[Text] = None unavailable_resources: List = list() # evaluate resources that would be gathered if pods were found @@ -284,6 +285,13 @@ def gather_details(self, kubectl: KubectlInterface, # determine the ingress controller for the deployment # this will help to evaluate which resources should be considered "unavailable" ingress_controller = ingress_util.determine_ingress_controller(resource_cache) + if ( + ingress_controller == SupportedIngress.Controllers.NGINX or + ingress_controller == SupportedIngress.Controllers.ISTIO or + ingress_controller == SupportedIngress.Controllers.OPENSHIFT or + ingress_controller == SupportedIngress.Controllers.CONTOUR + ): + ingress_version = ingress_util.get_ingress_version(kubectl) # determine if any resource types for which caching was attempted were unavailable # if at least one is unavailable, a message will be displayed saying that components may not be complete @@ -361,6 +369,9 @@ def gather_details(self, kubectl: KubectlInterface, # create a key to mark the determined ingress controller for the cluster: str|None k8s_details_dict[Keys.Kubernetes.INGRESS_CTRL]: Optional[Text] = ingress_controller + # create a key to mark the determined ingress version for the cluster: str|None + k8s_details_dict[Keys.Kubernetes.INGRESS_VER]: Optional[Text] = ingress_version + # create a key to mark the namespace evaluated for this report: str|None k8s_details_dict[Keys.Kubernetes.NAMESPACE] = kubectl.get_namespace() diff --git a/deployment_report/templates/viya_deployment_report.html.j2 b/deployment_report/templates/viya_deployment_report.html.j2 index 57f995d..b8a1842 100644 --- a/deployment_report/templates/viya_deployment_report.html.j2 +++ b/deployment_report/templates/viya_deployment_report.html.j2 @@ -89,10 +89,6 @@ Namespace {{ report_data.kubernetes.namespace | default("could not be determined") }} - - Ingress Controller - {{ report_data.kubernetes.ingressController | default("could not be determined") }} - Cadence Version {{ report_data.kubernetes.cadenceInfo | default("could not be determined") }} @@ -104,6 +100,7 @@

Versions

+ Kubernetes @@ -126,6 +123,21 @@
{{ report_data.kubernetes.versions.serverVersion.platform }}
+ Ingress + + + + + + + + {% if report_data.kubernetes.ingressVersion is not none and report_data.kubernetes.ingressVersion | length > 0 %} + + {% else %} + + {% endif %} + +
Controller{{ report_data.kubernetes.ingressController | default("could not be determined") }}
Version{{ report_data.kubernetes.ingressVersion}}Unavailable
{# Cluster Overview: Kubernetes Versions Accordion #} diff --git a/deployment_report/test/data/expected_usage_output.txt b/deployment_report/test/data/expected_usage_output.txt index df2ff61..ff68a37 100644 --- a/deployment_report/test/data/expected_usage_output.txt +++ b/deployment_report/test/data/expected_usage_output.txt @@ -1,4 +1,5 @@ -usage: viya-ark.py deployment-report [-h] [-d] [-k KUBECTL_GLOBAL_OPTS] [-l] +usage: viya-ark.py deployment-report [-h] [-d] [-i INGRESS_NAMESPACE] + [-k KUBECTL_GLOBAL_OPTS] [-l] [-n NAMESPACE] [-o OUTPUT_DIR] [-r] Generate a deployment report of SAS components for a target Kubernetes @@ -7,6 +8,8 @@ environment. optional arguments: -h, --help show this help message and exit -d, --data-file-only Generate only the JSON-formatted data. + -i INGRESS_NAMESPACE, --ingress-namespace INGRESS_NAMESPACE + Ingress namespace in the target cluster. -k KUBECTL_GLOBAL_OPTS, --kubectl-global-opts KUBECTL_GLOBAL_OPTS Any kubectl global options to use with all executions (excluding namespace, which should be set using -n, diff --git a/viya_ark_library/k8s/sas_k8s_ingress.py b/viya_ark_library/k8s/sas_k8s_ingress.py index 91aa516..2ab13a6 100644 --- a/viya_ark_library/k8s/sas_k8s_ingress.py +++ b/viya_ark_library/k8s/sas_k8s_ingress.py @@ -27,6 +27,10 @@ class Controllers(object): ISTIO = "Istio" NGINX = "NGINX" OPENSHIFT = "OpenShift" + NS_CONTOUR = "projectcontour" + NS_ISTIO = "istio-system" + NS_NGINX = "ingress-nginx" + NS_OPENSHIFT = "openshift-ingress-operator" @staticmethod def get_ingress_controller_to_resource_types_map() -> Dict[Text, List[Text]]: diff --git a/viya_ark_library/k8s/sas_kubectl.py b/viya_ark_library/k8s/sas_kubectl.py index eade64a..981bf98 100644 --- a/viya_ark_library/k8s/sas_kubectl.py +++ b/viya_ark_library/k8s/sas_kubectl.py @@ -18,6 +18,7 @@ from viya_ark_library.k8s.sas_k8s_errors import NamespaceNotFoundError from viya_ark_library.k8s.sas_k8s_objects import KubernetesAvailableResourceTypes, KubernetesMetrics, KubernetesResource from viya_ark_library.k8s.sas_kubectl_interface import KubectlInterface +from viya_ark_library.k8s.sas_k8s_ingress import SupportedIngress # header values used in retrieving values returned by kubectl _HEADER_NAME_ = "NAME" @@ -28,6 +29,9 @@ _HEADER_KIND_ = "KIND" _HEADER_VERBS_ = "VERBS" +# constants values +_DO_MSG_ = "command_fail_ignored" + class Kubectl(KubectlInterface): """ @@ -35,7 +39,8 @@ class Kubectl(KubectlInterface): execute requests to a Kubernetes cluster via functionality provided by the command-line utility. """ - def __init__(self, executable: Text = "kubectl", namespace: Text = None, global_opts: Text = "") -> None: + def __init__(self, executable: Text = "kubectl", namespace: Text = None, global_opts: Text = "", + ingress_namespace: Text = None) -> None: """ Constructor for Kubectl class. @@ -73,6 +78,28 @@ def __init__(self, executable: Text = "kubectl", namespace: Text = None, global_ # get the list of all namespaces existing_namespaces: List[KubernetesResource] = self.get_resources("namespaces") + # initialize values + self.ingress_ns = None + + # get ingress namespace + if ingress_namespace is not None: + ns: AnyStr = self.do("get namespace " + ingress_namespace, ignore_errors=True, warning=False) + if len(ns) > 0: + raise NamespaceNotFoundError( + f"The ingress namespace [{ingress_namespace}] was not found in the target environment.") + self.ingress_ns = ingress_namespace + else: + for x in range(len(existing_namespaces)): + ns: AnyStr = existing_namespaces[x].get_name() + if ( + ns == SupportedIngress.Controllers.NS_CONTOUR or + ns == SupportedIngress.Controllers.NS_ISTIO or + ns == SupportedIngress.Controllers.NS_NGINX or + ns == SupportedIngress.Controllers.NS_OPENSHIFT + ): + self.ingress_ns = ns + break + # loop over all existing namespaces and check for the given namespace for existing_namespace in existing_namespaces: if existing_namespace.get_name() == namespace: @@ -125,8 +152,8 @@ def __init__(self, executable: Text = "kubectl", namespace: Text = None, global_ def get_namespace(self) -> Text: return self.namespace - def do(self, command: Text, ignore_errors: bool = False, success_rcs: Optional[List[int]] = None) \ - -> AnyStr: + def do(self, command: Text, ignore_errors: bool = False, success_rcs: Optional[List[int]] = None, + warning: bool = True) -> AnyStr: # set default return code list if one was not provided if success_rcs is None: success_rcs = [0] @@ -145,8 +172,12 @@ def do(self, command: Text, ignore_errors: bool = False, success_rcs: Optional[L if not ignore_errors: raise CalledProcessError(returncode=rc, cmd=f"{self.exec} {command}", output=stdout, stderr=stderr) else: - print(f"WARNING: Error encountered executing: {self.exec} {command} " - f"(rc: {rc} | stdout: {stdout} | stderr: {stderr})") + if warning: + print(f"WARNING: Error encountered executing: {self.exec} {command} " + f"(rc: {rc} | stdout: {stdout} | stderr: {stderr})") + else: + if len(stdout) == 0: + stdout = _DO_MSG_ # return the stdout return stdout diff --git a/viya_ark_library/k8s/test_impl/sas_kubectl_test.py b/viya_ark_library/k8s/test_impl/sas_kubectl_test.py index 6412a2c..4c6add9 100644 --- a/viya_ark_library/k8s/test_impl/sas_kubectl_test.py +++ b/viya_ark_library/k8s/test_impl/sas_kubectl_test.py @@ -88,6 +88,7 @@ class Values(object): CADENCEINFO: Text = "Fast R/TR 2020 (20201214.1607958443388)" DB_External: Text = "External" DB_Internal: Text = "Internal" + INGRESS_NAMESPACE: Text = "" # Component: prometheus COMPONENT_PROMETHEUS_DEPLOYMENT_NAME: Text = "pushgateway-test-prometheus-pushgateway" @@ -435,7 +436,8 @@ def __init__(self, include_metrics: bool = True, include_non_namespaced_resources: bool = True, namespace: Text = Values.NAMESPACE, - simulate_empty_deployment: bool = False) -> None: + simulate_empty_deployment: bool = False, + ingress_ns: Text = Values.INGRESS_NAMESPACE) -> None: """ Constructor for the KubectlTest implementation. @@ -452,6 +454,7 @@ def __init__(self, self.include_non_namespaced_resources = include_non_namespaced_resources self.namespace = namespace self.simulate_empty_deployment = simulate_empty_deployment + self.ingress_ns = ingress_ns def get_namespace(self) -> Text: return self.namespace @@ -492,6 +495,9 @@ def api_resources(self, ignore_errors: bool = False) -> KubernetesAvailableResou def api_versions(self, ignore_errors: bool = False) -> List: return KubectlTest._load_response_data(_API_VERSIONS_DATA_) + def ingress_version(self) -> Optional[AnyStr]: + pass + def can_i(self, action: Text, all_namespaces: bool = False, ignore_errors: bool = False) -> bool: # this method is not functional in the testing implementation return False From 577601efa32360b776e159fe77884abc60782579 Mon Sep 17 00:00:00 2001 From: Amy Ho <39809594+cuddlehub@users.noreply.github.com> Date: Mon, 11 Jul 2022 12:55:58 -0400 Subject: [PATCH 02/20] (#123) Request to include version information for ingress controller (#175) --- deployment_report/model/utils/ingress_util.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/deployment_report/model/utils/ingress_util.py b/deployment_report/model/utils/ingress_util.py index 7004099..1d2ae00 100644 --- a/deployment_report/model/utils/ingress_util.py +++ b/deployment_report/model/utils/ingress_util.py @@ -120,12 +120,13 @@ def get_ingress_version(kubectl: KubectlInterface) -> Optional[Text]: """ version: Text = "" + getpod_cmd: AnyStr = "get pods -n " + kubectl.ingress_ns + \ + " --field-selector=status.phase==Running" + \ + " -o jsonpath='{.items[0].metadata.name}'" if kubectl.ingress_ns == SupportedIngress.Controllers.NS_NGINX: - podname: AnyStr = kubectl.do("get pods -n " + - kubectl.ingress_ns + - " -l app.kubernetes.io/component=controller" + - " -o jsonpath='{.items[0].metadata.name}'") + podname: AnyStr = kubectl.do(getpod_cmd + + " -l app.kubernetes.io/component=controller") version_str: AnyStr = kubectl.do("exec -it " + podname.decode() + " -n " + kubectl.ingress_ns + " -- /nginx-ingress-controller --version") @@ -137,18 +138,16 @@ def get_ingress_version(kubectl: KubectlInterface) -> Optional[Text]: version = version + ", " + v.split()[-1] elif kubectl.ingress_ns == SupportedIngress.Controllers.NS_ISTIO: - podname: AnyStr = kubectl.do("get pods -n " + kubectl.ingress_ns + - " -l app=istiod" + - " -o jsonpath='{.items[0].metadata.name}'") + podname: AnyStr = kubectl.do(getpod_cmd + + " -l app=istiod") version_str: AnyStr = kubectl.do("exec -it " + podname.decode() + " -n " + kubectl.ingress_ns + " -- pilot-discovery version --short") version = version_str.decode() elif kubectl.ingress_ns == SupportedIngress.Controllers.NS_OPENSHIFT: - podname: AnyStr = kubectl.do("get pods -n " + kubectl.ingress_ns + - " -l name=ingress-operator" + - " -o jsonpath='{.items[0].metadata.name}'") + podname: AnyStr = kubectl.do(getpod_cmd + + " -l name=ingress-operator") version_str: AnyStr = kubectl.do("get pod " + podname.decode() + " -n " + kubectl.ingress_ns + " -o jsonpath=\"{.spec.containers[].env[" + @@ -156,10 +155,8 @@ def get_ingress_version(kubectl: KubectlInterface) -> Optional[Text]: version = version_str.decode() elif kubectl.ingress_ns == SupportedIngress.Controllers.NS_CONTOUR: - podname: AnyStr = kubectl.do("get pods -n " + - kubectl.ingress_ns + - " -l app=envoy" + - " -o jsonpath='{.items[0].metadata.name}'") + podname: AnyStr = kubectl.do(getpod_cmd + + " -l app=envoy") version_str: AnyStr = kubectl.do("get pod " + podname.decode() + " -n " + kubectl.ingress_ns + " -o jsonpath='{.spec.containers[*].image}'") From 0ce1d0a4a041334f9a93db62e3ce57ff0872db7a Mon Sep 17 00:00:00 2001 From: Amy Ho <39809594+cuddlehub@users.noreply.github.com> Date: Mon, 11 Jul 2022 15:53:35 -0400 Subject: [PATCH 03/20] (#123) Request to include version information for ingress controller (#176) --- deployment_report/model/utils/ingress_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment_report/model/utils/ingress_util.py b/deployment_report/model/utils/ingress_util.py index 1d2ae00..7f1193b 100644 --- a/deployment_report/model/utils/ingress_util.py +++ b/deployment_report/model/utils/ingress_util.py @@ -122,7 +122,7 @@ def get_ingress_version(kubectl: KubectlInterface) -> Optional[Text]: version: Text = "" getpod_cmd: AnyStr = "get pods -n " + kubectl.ingress_ns + \ " --field-selector=status.phase==Running" + \ - " -o jsonpath='{.items[0].metadata.name}'" + " -o jsonpath=\"{.items[0].metadata.name}\"" if kubectl.ingress_ns == SupportedIngress.Controllers.NS_NGINX: podname: AnyStr = kubectl.do(getpod_cmd + @@ -159,7 +159,7 @@ def get_ingress_version(kubectl: KubectlInterface) -> Optional[Text]: " -l app=envoy") version_str: AnyStr = kubectl.do("get pod " + podname.decode() + " -n " + kubectl.ingress_ns + - " -o jsonpath='{.spec.containers[*].image}'") + " -o jsonpath=\"{.spec.containers[*].image}\"") version_list: List = version_str.decode().split(' ') for v in version_list: if version: From 392ba520547857ab907f0e6a2f1dccb02abae0ea Mon Sep 17 00:00:00 2001 From: Kevin Lingle <36995745+kevinlinglesas@users.noreply.github.com> Date: Wed, 5 Oct 2022 20:09:21 -0400 Subject: [PATCH 04/20] Update flake8 usage Output flake8 and pytest versions after installation Add --extend-ignore=E275 to ignore rule for adding whitespace after keywords. Appears to have been introduced as a default with flake8 5.x --- .github/workflows/ci-validation.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-validation.yml b/.github/workflows/ci-validation.yml index 081e252..357609c 100644 --- a/.github/workflows/ci-validation.yml +++ b/.github/workflows/ci-validation.yml @@ -23,11 +23,13 @@ jobs: run: | python -m pip install --upgrade pip pip install flake8 pytest + flake8 --version + pytest --version if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 . --count --max-line-length 120 --show-source --statistics + flake8 . --count --max-line-length 120 --show-source --statistics --extend-ignore=E275 - name: Test with pytest run: | pytest From f29401f7c41a6ea65a6b5d77371fac007445b537 Mon Sep 17 00:00:00 2001 From: lasiva Date: Thu, 27 Oct 2022 17:33:22 -0400 Subject: [PATCH 05/20] (#181) Update README for Open Source Kubernetes --- pre_install_report/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index 86fcb85..12c7d63 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -26,7 +26,8 @@ The requirements per offering are detailed in the _Hardware and Resource Require The required Memory and vCPUs sizes depend on the instance type used for the node. Your required aggregates must be specified in the following file -/viya4-ark/pre_install_report/viya_deployment_settings.ini, example: +/viya4-ark/pre_install_report/viya_deployment_settings.ini +Example: ``` # Total Memory of all worker Nodes in GB. Sum of the Memory on all active node required to deploy a specific offering. # Set value for required for offering @@ -37,13 +38,13 @@ VIYA_MIN_AGGREGATE_WORKER_CPU_CORES=56 ``` If calculated aggregate memory is less than a percentage (85%) of VIYA_MIN_AGGREGATE_WORKER_MEMORY then the tool will flag a memory issue. -If calculated aggregate vCPUs is less than VIYA_MIN_AGGREGATE_WORKER_CPU_CORES then it the tool will flag a CPU issue. +If calculated aggregate vCPUs is less than VIYA_MIN_AGGREGATE_WORKER_CPU_CORES then the tool will flag a CPU issue. SAS recommends using the SAS Viya 4 Infrastructure as Code (IaC) tools to create a cluster. -Refer to the following IaC repositories for [Microsoft Azure](https://github.com/sassoftware/viya4-iac-azure), [AWS](https://github.com/sassoftware/viya4-iac-aws]) or [GCP](https://github.com/sassoftware/viya4-iac-gcp). -For OpenShift refer to the documentation in SAS® Viya® Operations [OpenShift](https://go.documentation.sas.com/doc/en/itopscdc/v_019/itopssr/n1ika6zxghgsoqn1mq4bck9dx695.htm#p1c8bxlbu0gzuvn1e75nck1yozcn) -**Example**: Setting for aggregate Memory and vCPU for deployment based on documentation in SAS Viya Operations under System Requirements -in the Hardware and Resource Requirements section. See Sizing Recommendations for Microsoft Azure. +Refer to the following IaC repositories for [Microsoft Azure](https://github.com/sassoftware/viya4-iac-azure), [AWS](https://github.com/sassoftware/viya4-iac-aws]), [GCP](https://github.com/sassoftware/viya4-iac-gcp) or [Open Source Kubernetes](https://github.com/sassoftware/viya4-iac-k8s) +For OpenShift refer to the documentation in SAS® Viya® Operations [OpenShift](https://go.documentation.sas.com/doc/en/itopscdc/v_019/itopssr/n1ika6zxghgsoqn1mq4bck9dx695.htm#p1c8bxlbu0gzuvn1e75nck1yozcn) + +Example: See Sizing Recommendations for a Microsoft Azure deployment under [Hardware and Resource Requirements](https://go.documentation.sas.com/doc/en/itopscdc/v_028/itopssr/n0ampbltwqgkjkn1j3qogztsbbu0.htm). You can use this information to calculate VIYA_MIN_AGGREGATE_WORKER_MEMORY and VIYA_MIN_AGGREGATE_WORKER_CPU_CORES. | Offering | CAS Node(s) | System Node | Nodes in User Node Pool(s) | | ------------------------- |------------- | --------- | -------------| From 3588571c43fe7b01a00d9145477cf3022ceaf0e9 Mon Sep 17 00:00:00 2001 From: lasiva Date: Thu, 27 Oct 2022 18:07:51 -0400 Subject: [PATCH 06/20] (#181) Update Report messages for Open Source Kubernetes --- .../templates/report_template_viya_pre_install_check.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pre_install_report/templates/report_template_viya_pre_install_check.j2 b/pre_install_report/templates/report_template_viya_pre_install_check.j2 index 8bcb8af..c912ceb 100644 --- a/pre_install_report/templates/report_template_viya_pre_install_check.j2 +++ b/pre_install_report/templates/report_template_viya_pre_install_check.j2 @@ -161,8 +161,8 @@ -

** SAS recommends using the IaC tools to create the cluster. See SAS Viya 4 Infrastructure as Code (IaC) project for Microsoft Azure, AWS, and GCP. Refer to SAS Viya Documentation for OpenShift. -

** Also refer to SAS Viya Documentation for Sizing Recommendations for Microsoft Azure, AWS, GCP and OpenShift

+

** SAS recommends using the IaC tools to create the cluster. See SAS Viya 4 Infrastructure as Code (IaC) project for Microsoft Azure, AWS, GCP and Open Source Kubernetes. Refer to SAS Viya Documentation for OpenShift. +

** Also refer to SAS Viya Documentation for Sizing Recommendations for Microsoft Azure, AWS, GCP, Open Source Kubernetes and OpenShift
From 89d95243c5477fcfde9c95ba0fac160b1891785b Mon Sep 17 00:00:00 2001 From: Amy Ho <39809594+cuddlehub@users.noreply.github.com> Date: Fri, 28 Oct 2022 13:41:22 -0400 Subject: [PATCH 07/20] (#179) Request: crunchydata 5 support (#180) * (#179) Request: crunchydata 5 support * (#179) Request: crunchydata 5 support --- deployment_report/model/utils/config_util.py | 52 +- .../model/utils/test/test_config_util.py | 16 +- .../k8s/k8s_resource_type_values.py | 5 +- .../api_resources_ingress_none.json | 18 + ...ers.postgres-operator.crunchydata.com.json | 565 ++++++++++++++++++ 5 files changed, 653 insertions(+), 3 deletions(-) create mode 100644 viya_ark_library/k8s/test_impl/response_data/resources_postgresclusters.postgres-operator.crunchydata.com.json diff --git a/deployment_report/model/utils/config_util.py b/deployment_report/model/utils/config_util.py index 4f50e21..31ec686 100644 --- a/deployment_report/model/utils/config_util.py +++ b/deployment_report/model/utils/config_util.py @@ -4,7 +4,7 @@ # ### Author: SAS Institute Inc. ### #################################################################### # ### -# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA. ### +# Copyright (c) 2022, SAS Institute Inc., Cary, NC, USA. ### # All Rights Reserved. ### # SPDX-License-Identifier: Apache-2.0 ### # ### @@ -47,6 +47,7 @@ _UNSUPPORTED_DB_ = "Unsupported" _CRUNCH_DB_ = "crunch" _CRUNCHDATA_DB_ = "sas-crunchy-data-" +_CRUNCH5DATA_DB_ = "sas-crunchy-platform-" _ALT_DB_ = "sas.database.alternative.databaseServerName" _UNAVAIL_DB_ = "not available" @@ -116,6 +117,10 @@ def get_db_info(resource_cache: Dict) -> Dict: if not resource_cache: return None + if ResourceTypeValues.SAS_CRUNCHYCLUSTERS in resource_cache.keys(): + pgclusters: Dict = resource_cache[ResourceTypeValues.SAS_CRUNCHYCLUSTERS][ITEMS_KEY] + db_dict = _get_db_info_v3(pgclusters) + if ResourceTypeValues.SAS_PGCLUSTERS in resource_cache.keys(): pgclusters: Dict = resource_cache[ResourceTypeValues.SAS_PGCLUSTERS][ITEMS_KEY] db_dict = _get_db_info_v2(pgclusters) @@ -188,6 +193,51 @@ def _get_db_info_v1(config_maps: Dict) -> Dict: return db_dict +def _get_db_info_v3(pgclusters: Dict) -> Dict: + """ + Returns the db information of the targeted SAS deployment. + + :param pgclusters: The pgclusters resource to evaluate. + :return: A dictionary representing the db information of the targeted SAS deployment. + """ + # initialize the return value + db_dict: Dict = dict() + for key in pgclusters: + try: + resource_definition = pgclusters[key][Keys.ResourceDetails.RESOURCE_DEFINITION] + + db_name: Optional[Text] = resource_definition.get_name() + db_data: Optional[Dict] = resource_definition.get_spec() + + if not db_data: + continue + + dbs: Dict = dict() + + try: + if (db_data['port']): + dbs = {Keys.DatabaseDetails.DBTYPE: _INTERNAL_DB_} + else: + dbs = { + Keys.DatabaseDetails.DBTYPE: _EXTERNAL_DB_, + Keys.DatabaseDetails.DBCONN: _UNAVAIL_DB_ + } + except KeyError: + continue + + if dbs: + # convert db_name to be aligned + if _CRUNCH5DATA_DB_ in db_name: + db_name = db_name.replace(_CRUNCH5DATA_DB_, "") + + db_dict[db_name] = dbs + + except KeyError: + continue + + return db_dict + + def _get_db_info_v2(pgclusters: Dict) -> Dict: """ Returns the db information of the targeted SAS deployment. diff --git a/deployment_report/model/utils/test/test_config_util.py b/deployment_report/model/utils/test/test_config_util.py index 8021b92..b589ef9 100644 --- a/deployment_report/model/utils/test/test_config_util.py +++ b/deployment_report/model/utils/test/test_config_util.py @@ -4,7 +4,7 @@ # ### Author: SAS Institute Inc. ### #################################################################### # ### -# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA. ### +# Copyright (c) 2022, SAS Institute Inc., Cary, NC, USA. ### # All Rights Reserved. ### # SPDX-License-Identifier: Apache-2.0 ### # ### @@ -62,6 +62,20 @@ def test_get_db_info_v2(no_ingress_simulation_fixture: conftest.DSA): assert db_dict[config_util._DBNAME_POSTGRES_][Keys.DatabaseDetails.DBTYPE] == KubectlTest.Values.DB_External +@pytest.mark.usefixtures(conftest.NO_INGRESS_SIMULATION_FIXTURE) +def test_get_db_info_v3(no_ingress_simulation_fixture: conftest.DSA): + """ + This test verifies that the provided db data is returned + """ + # get the resource cache + resource_cache: Dict = no_ingress_simulation_fixture.resource_cache() + + pgclusters: Dict = resource_cache[ResourceTypeValues.SAS_CRUNCHYCLUSTERS][ITEMS_KEY] + db_dict: Dict = config_util._get_db_info_v3(pgclusters=pgclusters) + + assert db_dict[config_util._DBNAME_POSTGRES_][Keys.DatabaseDetails.DBTYPE] == KubectlTest.Values.DB_Internal + + @pytest.mark.usefixtures(conftest.NO_INGRESS_SIMULATION_FIXTURE) def test_get_db_info_v1(no_ingress_simulation_fixture: conftest.DSA): """ diff --git a/viya_ark_library/k8s/k8s_resource_type_values.py b/viya_ark_library/k8s/k8s_resource_type_values.py index c7f0245..b8a5726 100644 --- a/viya_ark_library/k8s/k8s_resource_type_values.py +++ b/viya_ark_library/k8s/k8s_resource_type_values.py @@ -4,7 +4,7 @@ # ### Author: SAS Institute Inc. ### #################################################################### # ### -# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA. ### +# Copyright (c) 2022, SAS Institute Inc., Cary, NC, USA. ### # All Rights Reserved. ### # SPDX-License-Identifier: Apache-2.0 ### # ### @@ -66,6 +66,9 @@ class KubernetesResourceTypeValues(object): SAS_GROUP_WEBINFDSVR_SAS_COM = "webinfdsvr.sas.com" SAS_PGCLUSTERS = f"pgclusters.{SAS_GROUP_WEBINFDSVR_SAS_COM}" + SAS_GROUP_CRUNCHYDATA_COM = "postgres-operator.crunchydata.com" + SAS_CRUNCHYCLUSTERS = f"postgresclusters.{SAS_GROUP_CRUNCHYDATA_COM}" + ############################## # Third-Party Resources ############################## diff --git a/viya_ark_library/k8s/test_impl/response_data/api_resources_ingress_none.json b/viya_ark_library/k8s/test_impl/response_data/api_resources_ingress_none.json index d558f9f..ea7ed8f 100644 --- a/viya_ark_library/k8s/test_impl/response_data/api_resources_ingress_none.json +++ b/viya_ark_library/k8s/test_impl/response_data/api_resources_ingress_none.json @@ -237,5 +237,23 @@ "watch" ], "version": "v1" + }, + "postgresclusters.postgres-operator.crunchydata.com": { + "group": "postgres-operator.crunchydata.com", + "kind": "PostgresCluster", + "name": "postgresclusters", + "namespaced": true, + "shortname": "", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "version": "v1" } } diff --git a/viya_ark_library/k8s/test_impl/response_data/resources_postgresclusters.postgres-operator.crunchydata.com.json b/viya_ark_library/k8s/test_impl/response_data/resources_postgresclusters.postgres-operator.crunchydata.com.json new file mode 100644 index 0000000..257e5bb --- /dev/null +++ b/viya_ark_library/k8s/test_impl/response_data/resources_postgresclusters.postgres-operator.crunchydata.com.json @@ -0,0 +1,565 @@ +[ + { + "apiVersion": "postgres-operator.crunchydata.com/v1beta1", + "kind": "PostgresCluster", + "metadata": { + "annotations": { + "kubectl.kubernetes.io/last-applied-configuration": "" + }, + "creationTimestamp": "2022-09-29T18:07:44Z", + "finalizers": [ + "postgres-operator.crunchydata.com/finalizer" + ], + "generation": 1, + "labels": { + "sas.com/admin": "namespace", + "sas.com/deployment": "sas-viya" + }, + "name": "sas-crunchy-platform-postgres", + "namespace": "test", + "resourceVersion": "62473", + "uid": "2ff47dfe-bc15-4bcc-be82-9255bf5dd11e" + }, + "spec": { + "backups": { + "pgbackrest": { + "global": { + "repo1-retention-archive": "1", + "repo1-retention-archive-type": "incr", + "repo1-retention-full": "1" + }, + "image": "test.sas.com/viya-4-x64_oci_linux_2-docker-testready/sas-crunchy5-pgbackrest:1.0.2-20220901.1662054072169", + "jobs": { + "affinity": { + "nodeAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "preference": { + "matchExpressions": [ + { + "key": "workload.sas.com/class", + "operator": "In", + "values": [ + "stateful" + ] + } + ] + }, + "weight": 100 + }, + { + "preference": { + "matchExpressions": [ + { + "key": "workload.sas.com/class", + "operator": "NotIn", + "values": [ + "compute", + "cas", + "stateless", + "connect" + ] + } + ] + }, + "weight": 50 + } + ], + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.azure.com/mode", + "operator": "NotIn", + "values": [ + "system" + ] + } + ] + } + ] + } + } + }, + "resources": { + "limits": { + "cpu": "1", + "memory": "500Mi" + }, + "requests": { + "cpu": "100m", + "memory": "256Mi" + } + }, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "workload.sas.com/class", + "operator": "Equal", + "value": "stateful" + }, + { + "effect": "NoSchedule", + "key": "workload.sas.com/class", + "operator": "Equal", + "value": "stateless" + } + ] + }, + "repoHost": { + "affinity": { + "nodeAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "preference": { + "matchExpressions": [ + { + "key": "workload.sas.com/class", + "operator": "In", + "values": [ + "stateful" + ] + } + ] + }, + "weight": 100 + }, + { + "preference": { + "matchExpressions": [ + { + "key": "workload.sas.com/class", + "operator": "NotIn", + "values": [ + "compute", + "cas", + "stateless", + "connect" + ] + } + ] + }, + "weight": 50 + } + ], + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.azure.com/mode", + "operator": "NotIn", + "values": [ + "system" + ] + } + ] + } + ] + } + } + }, + "resources": { + "limits": { + "cpu": "1", + "memory": "500Mi" + }, + "requests": { + "cpu": "100m", + "memory": "256Mi" + } + }, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "workload.sas.com/class", + "operator": "Equal", + "value": "stateful" + }, + { + "effect": "NoSchedule", + "key": "workload.sas.com/class", + "operator": "Equal", + "value": "stateless" + } + ] + }, + "repos": [ + { + "name": "repo1", + "schedules": { + "full": "0 6 * * 0", + "incremental": "0 6 * * 1-6" + }, + "volume": { + "volumeClaimSpec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "128Gi" + } + } + } + } + } + ], + "sidecars": { + "pgbackrest": { + "resources": { + "limits": { + "cpu": "1", + "memory": "500Mi" + }, + "requests": { + "cpu": "100m", + "memory": "256Mi" + } + } + } + } + } + }, + "customReplicationTLSSecret": { + "name": "sas-crunchy-platform-postgres-replication-tls-secret" + }, + "customTLSSecret": { + "name": "sas-crunchy-platform-postgres-tls-secret" + }, + "image": "test.sas.com/viya-4-x64_oci_linux_2-docker-testready/sas-crunchy5-postgres:1.0.2-20220901.1662054048918", + "imagePullSecrets": [ + { + "name": "sas-image-pull-secrets-f82b98k5t2" + } + ], + "instances": [ + { + "affinity": { + "nodeAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "preference": { + "matchExpressions": [ + { + "key": "workload.sas.com/class", + "operator": "In", + "values": [ + "stateful" + ] + } + ] + }, + "weight": 100 + }, + { + "preference": { + "matchExpressions": [ + { + "key": "workload.sas.com/class", + "operator": "NotIn", + "values": [ + "compute", + "cas", + "stateless", + "connect" + ] + } + ] + }, + "weight": 50 + } + ], + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.azure.com/mode", + "operator": "NotIn", + "values": [ + "system" + ] + } + ] + } + ] + } + } + }, + "dataVolumeClaimSpec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "128Gi" + } + } + }, + "name": "", + "replicas": 3, + "resources": { + "limits": { + "cpu": "8", + "memory": "8Gi" + }, + "requests": { + "cpu": "150m", + "memory": "2Gi" + } + }, + "sidecars": { + "replicaCertCopy": { + "resources": { + "limits": { + "cpu": "1", + "memory": "500Mi" + }, + "requests": { + "cpu": "100m", + "memory": "256Mi" + } + } + } + }, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "workload.sas.com/class", + "operator": "Equal", + "value": "stateful" + }, + { + "effect": "NoSchedule", + "key": "workload.sas.com/class", + "operator": "Equal", + "value": "stateless" + } + ] + } + ], + "metadata": { + "annotations": { + "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" + }, + "labels": { + "sas.com/deployment": "sas-viya", + "sas.com/zero-scale-phase": "never", + "workload.sas.com/class": "stateful" + } + }, + "monitoring": { + "pgmonitor": { + "exporter": { + "image": "test.sas.com/viya-4-x64_oci_linux_2-docker-testready/sas-crunchy5-postgres-exporter:1.0.3-20220926.1664209426020", + "resources": { + "limits": { + "cpu": "1", + "memory": "500Mi" + }, + "requests": { + "cpu": "100m", + "memory": "256Mi" + } + } + } + } + }, + "patroni": { + "dynamicConfiguration": { + "postgresql": { + "parameters": { + "log_truncate_on_rotation": true, + "logging_collector": true, + "maintenance_work_mem": "128MB", + "max_connections": 1280, + "max_prepared_transactions": 1280, + "max_wal_senders": 6, + "max_wal_size": "6GB", + "shared_buffers": "1GB", + "wal_buffers": "16MB", + "wal_keep_segments": 500, + "wal_level": "minimal", + "wal_log_hints": true, + "work_mem": "16MB" + }, + "use_pg_rewind": true + } + }, + "leaderLeaseDurationSeconds": 30, + "port": 8008, + "syncPeriodSeconds": 10 + }, + "port": 5432, + "postgresVersion": 12, + "userInterface": { + "pgAdmin": { + "affinity": { + "nodeAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "preference": { + "matchExpressions": [ + { + "key": "workload.sas.com/class", + "operator": "In", + "values": [ + "stateful" + ] + } + ] + }, + "weight": 100 + }, + { + "preference": { + "matchExpressions": [ + { + "key": "workload.sas.com/class", + "operator": "NotIn", + "values": [ + "compute", + "cas", + "stateless", + "connect" + ] + } + ] + }, + "weight": 50 + } + ], + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.azure.com/mode", + "operator": "NotIn", + "values": [ + "system" + ] + } + ] + } + ] + } + } + }, + "dataVolumeClaimSpec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "1Gi" + } + } + }, + "image": "test.sas.com/viya-4-x64_oci_linux_2-docker-testready/sas-crunchy5-pgadmin:1.0.2-20220901.1662054069256", + "replicas": 1, + "resources": { + "limits": { + "cpu": "500m", + "memory": "500Mi" + }, + "requests": { + "cpu": "50m", + "memory": "50Mi" + } + }, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "workload.sas.com/class", + "operator": "Equal", + "value": "stateful" + }, + { + "effect": "NoSchedule", + "key": "workload.sas.com/class", + "operator": "Equal", + "value": "stateless" + } + ] + } + }, + "users": [ + { + "name": "dbmsowner", + "options": "CREATEDB CREATEROLE SUPERUSER", + "password": { + "type": "AlphaNumeric" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastTransitionTime": "2022-09-29T18:15:23Z", + "message": "pgBackRest dedicated repository host is ready", + "observedGeneration": 1, + "reason": "RepoHostReady", + "status": "True", + "type": "PGBackRestRepoHostReady" + }, + { + "lastTransitionTime": "2022-09-29T18:15:29Z", + "message": "pgBackRest replica create repo is ready for backups", + "observedGeneration": 1, + "reason": "StanzaCreated", + "status": "True", + "type": "PGBackRestReplicaRepoReady" + }, + { + "lastTransitionTime": "2022-09-29T18:09:46Z", + "message": "pgBackRest replica creation is not currently possible", + "observedGeneration": 1, + "reason": "RepoBackupNotComplete", + "status": "False", + "type": "PGBackRestReplicaCreate" + } + ], + "databaseRevision": "559678bf8f", + "instances": [ + { + "name": "00", + "readyReplicas": 3, + "replicas": 3, + "updatedReplicas": 3 + } + ], + "monitoring": { + "exporterConfiguration": "74cd546f8b" + }, + "observedGeneration": 1, + "patroni": { + "systemIdentifier": "7148866271475249228" + }, + "pgbackrest": { + "repoHost": { + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "ready": true + }, + "repos": [ + { + "bound": true, + "name": "repo1", + "stanzaCreated": true, + "volume": "pvc-c32c11f1-3133-48a5-9e22-0760cc8ecf3f" + } + ] + }, + "proxy": { + "pgBouncer": { + "postgresRevision": "5c9966f6bc" + } + }, + "usersRevision": "5c65959f95" + } + } +] From 751d48f0825fb52fb419f28be4eb900d2edf0ebb Mon Sep 17 00:00:00 2001 From: lasiva Date: Fri, 28 Oct 2022 16:18:57 -0400 Subject: [PATCH 08/20] (#181) Update Report messages for Open Source Kubernetes --- .../templates/report_template_viya_pre_install_check.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pre_install_report/templates/report_template_viya_pre_install_check.j2 b/pre_install_report/templates/report_template_viya_pre_install_check.j2 index c912ceb..e607ac3 100644 --- a/pre_install_report/templates/report_template_viya_pre_install_check.j2 +++ b/pre_install_report/templates/report_template_viya_pre_install_check.j2 @@ -274,8 +274,8 @@ -

** SAS recommends using the IaC tools to create the cluster. See SAS Viya 4 Infrastructure as Code (IaC) project for Microsoft Azure, AWS, and GCP -

** Also refer to SAS Viya Documentation for Sizing Recommendations for Microsoft Azure, AWS and GCP

+

** SAS recommends using the IaC tools to create the cluster. See SAS Viya 4 Infrastructure as Code (IaC) project for Microsoft Azure, AWS, GCP and Open Source Kubernetes. Refer to SAS Viya Documentation for OpenShift. +

** Also refer to SAS Viya Documentation for Sizing Recommendations for Microsoft Azure, AWS, Open Source Kubernetes and OpenShift
From 1a52bd89a9ca3247b34bbb555746ac95548eb05b Mon Sep 17 00:00:00 2001 From: lasiva Date: Fri, 28 Oct 2022 17:24:44 -0400 Subject: [PATCH 09/20] (#181) Update Report messages for Open Source Kubernetes --- pre_install_report/library/pre_install_check.py | 4 +++- pre_install_report/library/utils/viya_messages.py | 2 ++ .../templates/report_template_viya_pre_install_check.j2 | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pre_install_report/library/pre_install_check.py b/pre_install_report/library/pre_install_check.py index 2cf15d5..e33d5aa 100644 --- a/pre_install_report/library/pre_install_check.py +++ b/pre_install_report/library/pre_install_check.py @@ -1092,7 +1092,9 @@ def generate_report(self, namespace_admin_permission_data=namespace_admin_permission_data.items(), ingress_data=ingress_data.items(), namespace_admin_permission_aggregate=ns_admin_permission_aggregate['Permissions'], - cluster_admin_permission_aggregate=cluster_admin_permission_aggregate['Permissions']) + cluster_admin_permission_aggregate=cluster_admin_permission_aggregate['Permissions'], + cluster_creation_info=viya_messages.CLUSTER_CREATION_INFO, + sizings_info=viya_messages.SIZINGS_INFO) print("Created: {}".format(report_file_path)) print("Created: {}".format(output_directory + _REPORT_LOG_NAME_TMPL_.format(file_timestamp))) diff --git a/pre_install_report/library/utils/viya_messages.py b/pre_install_report/library/utils/viya_messages.py index fae9e32..a206145 100644 --- a/pre_install_report/library/utils/viya_messages.py +++ b/pre_install_report/library/utils/viya_messages.py @@ -30,6 +30,8 @@ KUBECONF_FILE_ERROR = "ERROR: The file specified in the KUBECONFIG environment does not exist. " \ "Check that file {} exists." KUBERNETES_VERSION_ERROR = "Kubernetes version is missing or invalid: {}" +CLUSTER_CREATION_INFO = "** SAS recommends using the IaC tools to create the cluster. See SAS Viya 4 Infrastructure as Code (IaC) project for Microsoft Azure, AWS, GCP and Open Source Kubernetes. Refer to SAS Viya Documentation for OpenShift." +SIZINGS_INFO = "** Also refer to SAS Viya Documentation for Sizing Recommendations for Microsoft Azure, AWS, GCP, Open Source Kubernetes and OpenShift." # command line return codes # SUCCESS_RC_ = 0 diff --git a/pre_install_report/templates/report_template_viya_pre_install_check.j2 b/pre_install_report/templates/report_template_viya_pre_install_check.j2 index e607ac3..9903c12 100644 --- a/pre_install_report/templates/report_template_viya_pre_install_check.j2 +++ b/pre_install_report/templates/report_template_viya_pre_install_check.j2 @@ -161,8 +161,8 @@ -

** SAS recommends using the IaC tools to create the cluster. See SAS Viya 4 Infrastructure as Code (IaC) project for Microsoft Azure, AWS, GCP and Open Source Kubernetes. Refer to SAS Viya Documentation for OpenShift. -

** Also refer to SAS Viya Documentation for Sizing Recommendations for Microsoft Azure, AWS, GCP, Open Source Kubernetes and OpenShift

+

{{cluster_creation_info}}

+

{{sizings_info}}
@@ -274,8 +274,8 @@ -

** SAS recommends using the IaC tools to create the cluster. See SAS Viya 4 Infrastructure as Code (IaC) project for Microsoft Azure, AWS, GCP and Open Source Kubernetes. Refer to SAS Viya Documentation for OpenShift. -

** Also refer to SAS Viya Documentation for Sizing Recommendations for Microsoft Azure, AWS, Open Source Kubernetes and OpenShift

+

{{cluster_creation_info}}

+

{{sizings_info}}
From 6a47c2b1706fd578222377aa221017e0bbbf846e Mon Sep 17 00:00:00 2001 From: lasiva Date: Fri, 28 Oct 2022 18:08:24 -0400 Subject: [PATCH 10/20] (#181) Update Report messages for Open Source Kubernetes --- pre_install_report/library/utils/viya_messages.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pre_install_report/library/utils/viya_messages.py b/pre_install_report/library/utils/viya_messages.py index a206145..5720e69 100644 --- a/pre_install_report/library/utils/viya_messages.py +++ b/pre_install_report/library/utils/viya_messages.py @@ -30,8 +30,11 @@ KUBECONF_FILE_ERROR = "ERROR: The file specified in the KUBECONFIG environment does not exist. " \ "Check that file {} exists." KUBERNETES_VERSION_ERROR = "Kubernetes version is missing or invalid: {}" -CLUSTER_CREATION_INFO = "** SAS recommends using the IaC tools to create the cluster. See SAS Viya 4 Infrastructure as Code (IaC) project for Microsoft Azure, AWS, GCP and Open Source Kubernetes. Refer to SAS Viya Documentation for OpenShift." -SIZINGS_INFO = "** Also refer to SAS Viya Documentation for Sizing Recommendations for Microsoft Azure, AWS, GCP, Open Source Kubernetes and OpenShift." +CLUSTER_CREATION_INFO = "** SAS recommends using the IaC tools to create the cluster. See SAS Viya 4 " \ + "Infrastructure as Code (IaC) project for Microsoft Azure, AWS, GCP and " \ + "Open Source Kubernetes. Refer to SAS Viya Documentation for OpenShift." +SIZINGS_INFO = "** Also refer to SAS Viya Documentation for Sizing Recommendations for Microsoft Azure, AWS, " \ + "GCP, Open Source Kubernetes and OpenShift." # command line return codes # SUCCESS_RC_ = 0 From 9cfa9e9b0e62a447fffb78884c6218780e99f6c3 Mon Sep 17 00:00:00 2001 From: Fred Perry <69848070+FredPerrySAS@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:22:20 -0400 Subject: [PATCH 11/20] Update README.md Mostly changes to old URLs. --- pre_install_report/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index 12c7d63..cfab54c 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -25,9 +25,9 @@ The calculated aggregate number of vCPUs must equal or exceed the required aggre The requirements per offering are detailed in the _Hardware and Resource Requirements_ section of the SAS Viya Operations document. The required Memory and vCPUs sizes depend on the instance type used for the node. -Your required aggregates must be specified in the following file +Your required aggregates must be specified in the following file: /viya4-ark/pre_install_report/viya_deployment_settings.ini -Example: +Here is an example: ``` # Total Memory of all worker Nodes in GB. Sum of the Memory on all active node required to deploy a specific offering. # Set value for required for offering @@ -37,14 +37,14 @@ VIYA_MIN_AGGREGATE_WORKER_MEMORY=448G VIYA_MIN_AGGREGATE_WORKER_CPU_CORES=56 ``` -If calculated aggregate memory is less than a percentage (85%) of VIYA_MIN_AGGREGATE_WORKER_MEMORY then the tool will flag a memory issue. -If calculated aggregate vCPUs is less than VIYA_MIN_AGGREGATE_WORKER_CPU_CORES then the tool will flag a CPU issue. +If the calculated aggregate memory is less than a percentage (85%) of VIYA_MIN_AGGREGATE_WORKER_MEMORY then the tool will flag a memory issue. +If the calculated aggregate vCPUs is less than VIYA_MIN_AGGREGATE_WORKER_CPU_CORES then the tool will flag a CPU issue. SAS recommends using the SAS Viya 4 Infrastructure as Code (IaC) tools to create a cluster. Refer to the following IaC repositories for [Microsoft Azure](https://github.com/sassoftware/viya4-iac-azure), [AWS](https://github.com/sassoftware/viya4-iac-aws]), [GCP](https://github.com/sassoftware/viya4-iac-gcp) or [Open Source Kubernetes](https://github.com/sassoftware/viya4-iac-k8s) -For OpenShift refer to the documentation in SAS® Viya® Operations [OpenShift](https://go.documentation.sas.com/doc/en/itopscdc/v_019/itopssr/n1ika6zxghgsoqn1mq4bck9dx695.htm#p1c8bxlbu0gzuvn1e75nck1yozcn) +For OpenShift refer to the documentation in SAS® Viya® Operations [OpenShift](https://go.documentation.sas.com/doc/en/itopscdc/default/itopssr/n1ika6zxghgsoqn1mq4bck9dx695.htm#p1c8bxlbu0gzuvn1e75nck1yozcn) -Example: See Sizing Recommendations for a Microsoft Azure deployment under [Hardware and Resource Requirements](https://go.documentation.sas.com/doc/en/itopscdc/v_028/itopssr/n0ampbltwqgkjkn1j3qogztsbbu0.htm). You can use this information to calculate VIYA_MIN_AGGREGATE_WORKER_MEMORY and VIYA_MIN_AGGREGATE_WORKER_CPU_CORES. +Example: See Sizing Recommendations for a Microsoft Azure deployment under [Hardware and Resource Requirements](https://go.documentation.sas.com/doc/en/itopscdc/default/itopssr/n0ampbltwqgkjkn1j3qogztsbbu0.htm#p1wmvm5pzezbwxn1rjjftlfqmeiu). You can use this information to calculate VIYA_MIN_AGGREGATE_WORKER_MEMORY and VIYA_MIN_AGGREGATE_WORKER_CPU_CORES. | Offering | CAS Node(s) | System Node | Nodes in User Node Pool(s) | | ------------------------- |------------- | --------- | -------------| From 2c808cbad78030879731ec9c292fb04a7b7d8c58 Mon Sep 17 00:00:00 2001 From: Latha Sivakumar Date: Tue, 8 Nov 2022 10:30:21 -0500 Subject: [PATCH 12/20] (#184) K8s minimum version check --- .../library/pre_install_check.py | 68 ++++++++++-------- .../library/utils/viya_constants.py | 3 - .../library/utils/viya_messages.py | 2 +- pre_install_report/pre_install_report.py | 12 +++- .../test/test_pre_install_report.py | 69 ++++++++++--------- .../viya_deployment_settings.ini | 5 +- 6 files changed, 88 insertions(+), 71 deletions(-) diff --git a/pre_install_report/library/pre_install_check.py b/pre_install_report/library/pre_install_check.py index 2cf15d5..5489956 100644 --- a/pre_install_report/library/pre_install_check.py +++ b/pre_install_report/library/pre_install_check.py @@ -55,7 +55,7 @@ class ViyaPreInstallCheck(): The gathered data can be written to disk as an HTML report and a JSON file containing the gathered data. """ - def __init__(self, sas_logger: ViyaARKLogger, viya_kubelet_version_min, + def __init__(self, sas_logger: ViyaARKLogger, viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory): """ @@ -65,7 +65,8 @@ def __init__(self, sas_logger: ViyaARKLogger, viya_kubelet_version_min, self._kubectl: KubectlInterface = None self.sas_logger = sas_logger self.logger = self.sas_logger.get_logger() - self._viya_kubelet_version_min = viya_kubelet_version_min + self._viya_k8s_version_min = viya_k8s_version_min + self._validated_kubernetes_version_min = None self._viya_min_aggregate_worker_CPU_cores: Text = viya_min_aggregate_worker_CPU_cores self._viya_min_aggregate_worker_memory: Text = viya_min_aggregate_worker_memory self._calculated_aggregate_memory = None @@ -76,21 +77,26 @@ def __init__(self, sas_logger: ViyaARKLogger, viya_kubelet_version_min, def _parse_release_info(self, release_info): """ - This method checks that the format of the VIYA_KUBELET_VERSION_MIN specfied in the - user modifiable properies file is valid. + This method checks that the format of the VIYA_K8s_VERSION_MIN specified in the + user modifiable ini file is valid. - :param release_info: The minimum Kubelet version loaded from properties file. - :return tuple of major version, minor version, patch + :param release_info: The minimum K8s version loaded from ini file. + :return tuple of major version, minor version """ try: info = tuple(release_info.split(".")) - return info + if (len(info) == 2): + x = [int(i) for a, i in enumerate(info)] + self.logger.debug('release tuple to int {} '.format(x)) + k8s_min_rel_str = ''.join(release_info) + self._validated_kubernetes_version_min = k8s_min_rel_str + else: + print('****' + viya_messages.KUBELET_VERSION_ERROR) + sys.exit(viya_messages.BAD_OPT_RC_) + except ValueError: print(viya_messages.KUBELET_VERSION_ERROR) sys.exit(viya_messages.BAD_OPT_RC_) - if (len(info) != 3): - print(viya_messages.KUBELET_VERSION_ERROR) - sys.exit(viya_messages.BAD_OPT_RC_) def _validate_k8s_server_version(self, version): """ @@ -140,7 +146,10 @@ def _k8s_server_version_min(self): try: curr_version = semantic_version.Version(str(self._k8s_server_version)) - if(curr_version in semantic_version.SimpleSpec(viya_constants.MIN_K8S_SERVER_VERSION)): + self._parse_release_info(self._viya_k8s_version_min) + min_k8s_version = "<" + self._validated_kubernetes_version_min + + if(curr_version in semantic_version.SimpleSpec(min_k8s_version)): self.logger.error("This release of Kubernetes is not supported {}.{}.x" .format(str(curr_version.major), str(curr_version.minor))) @@ -683,26 +692,26 @@ def _check_memory_errors(self, global_data, total_capacity_memory, quantity_, ag global_data.append(aggregate_memory_data) return global_data - def _check_kubelet_errors(self, global_data, aggregate_kubelet_failures): + def _check_k8s_errors(self, global_data, aggregate_k8s_failures): """ - Check kubelet version against SAS kubelet version requirements + Check Server k8s version against SAS k8s version requirements global_data: list with global data about worker nodes retrieved - aggregate_kubelet_failures: count of kubelet version errors + aggregate_k8s_failures: count of k8s version errors return: updated global data about worker nodes retrieved """ - aggregate_kubelet_data = {} + aggregate_k8s_data = {} node_status_msg = "" if self._aggregate_nodeStatus_failures > 0: node_status_msg = " Check Node(s). All Nodes NOT in Ready Status." \ + ' Issues Found: ' + str(self._aggregate_nodeStatus_failures) - aggregate_kubelet_data.update({'aggregate_kubelet_failures': node_status_msg}) - if aggregate_kubelet_failures > 0: - aggregate_kubelet_data.update({'aggregate_kubelet_failures': - 'Check Kubelet Version on nodes.' + - ' Issues Found: ' + str(aggregate_kubelet_failures) + - '.' + node_status_msg}) - global_data.append(aggregate_kubelet_data) + aggregate_k8s_data.update({'aggregate_k8s_failures': node_status_msg}) + if aggregate_k8s_failures > 0: + aggregate_k8s_data.update({'aggregate_k8s_failures:': + 'Check K8s Version on nodes.' + + ' Issues Found: ' + str(aggregate_k8s_failures) + + '.' + node_status_msg}) + global_data.append(aggregate_k8s_data) return global_data @@ -795,7 +804,7 @@ def evaluate_nodes(self, nodes_data, global_data, cluster_info, quantity_): """ aggregate_cpu_failures = int(0) aggregate_memory_failures = int(0) - aggregate_kubelet_failures = int(0) + aggregate_k8s_failures = int(0) total_cpu_cores = float(0) total_capacity_memory = quantity_("0G") @@ -838,10 +847,10 @@ def evaluate_nodes(self, nodes_data, global_data, cluster_info, quantity_): else: self._set_status(1, node, 'kubeletversion') node['error']['kubeletversion'] = viya_constants.SET + ': ' + kubeletversion + ', ' + \ - str(viya_constants.EXPECTED) + ': ' + viya_constants.MIN_K8S_SERVER_VERSION[1:] + ' or later ' + str(viya_constants.EXPECTED) + ': ' + self._validated_kubernetes_version_min + ' or later ' - aggregate_kubelet_failures += 1 - self.logger.debug("aggregate_kubelet_failures {} ".format(str(aggregate_kubelet_failures))) + aggregate_k8s_failures += 1 + self.logger.debug("aggregate_k8s_failures {} ".format(str(aggregate_k8s_failures))) self.logger.debug("node kubeletversion{} ".format(pprint.pformat(node))) global_data = self._check_workers(global_data, nodes_data) @@ -850,7 +859,7 @@ def evaluate_nodes(self, nodes_data, global_data, cluster_info, quantity_): global_data = self._check_memory_errors(global_data, total_capacity_memory, quantity_, aggregate_memory_failures) - global_data = self._check_kubelet_errors(global_data, aggregate_kubelet_failures) + global_data = self._check_k8s_errors(global_data, aggregate_k8s_failures) global_data.append(nodes_data) global_data = self._update_k8s_version(global_data, self._k8s_server_version) @@ -876,7 +885,7 @@ def _get_cpu_units(self, node, key): def _set_status(self, status, node, key): """Set the status flag on dictionary object with node details - to indicate compliance with - cpu, memory kubelet version requirements + cpu, memory k8s version requirements status: status to be set on dict objectwith node details node: node dictionary object @@ -1055,6 +1064,9 @@ def get_calculated_aggregate_memory(self): def set_k8s_version(self, version: Text): self._k8s_server_version = version + def set_k8s_version_min(self, version: Text): + self._viya_k8s_version_min = version + def generate_report(self, global_data, master_data, diff --git a/pre_install_report/library/utils/viya_constants.py b/pre_install_report/library/utils/viya_constants.py index 74d8bfc..3643298 100644 --- a/pre_install_report/library/utils/viya_constants.py +++ b/pre_install_report/library/utils/viya_constants.py @@ -61,6 +61,3 @@ VIYA_PERCENTAGE_OF_INSTANCE = "85" MEMORY_WITHIN_RANGE = " Memory within Range" SERVER_K8S_VERSION = "Server_k8s_version" - -# Any versions below this minimum are not supported -MIN_K8S_SERVER_VERSION = '<1.20' diff --git a/pre_install_report/library/utils/viya_messages.py b/pre_install_report/library/utils/viya_messages.py index fae9e32..8cfe9b5 100644 --- a/pre_install_report/library/utils/viya_messages.py +++ b/pre_install_report/library/utils/viya_messages.py @@ -20,7 +20,7 @@ "to access the Kubernetes cluster." CHECK_NAMESPACE = ' ERROR: Check available permissions in namespace and if it is valid: ' LIMIT_ERROR = 'ERROR: The value in the viya_deployment_settings.ini file is not valid: {} = {}' -KUBELET_VERSION_ERROR = 'ERROR: Check the VIYA_KUBELET_VERSION_MIN value ' \ +KUBELET_VERSION_ERROR = 'ERROR: Check the VIYA_K8S_VERSION_MIN value ' \ 'specified in the pre_install_report/viya_deployment_settings.ini file' OPTION_ERROR = "ERROR: option {} not recognized" OPTION_VALUES_ERROR = "ERROR: Provide valid values for all required options. Check options -i, -p and -H." diff --git a/pre_install_report/pre_install_report.py b/pre_install_report/pre_install_report.py index f7467c7..2bbc752 100644 --- a/pre_install_report/pre_install_report.py +++ b/pre_install_report/pre_install_report.py @@ -252,10 +252,18 @@ def main(argv): check_limits = _read_config_file('viya_deployment_settings.ini') with LRPIndicator(enter_message="Gathering facts"): - sas_pre_check_report: ViyaPreInstallCheck = \ - ViyaPreInstallCheck(sas_logger, check_limits['items']['VIYA_KUBELET_VERSION_MIN'], + try: + sas_pre_check_report: ViyaPreInstallCheck = \ + ViyaPreInstallCheck(sas_logger, check_limits['items']['VIYA_K8S_VERSION_MIN'], check_limits['items']['VIYA_MIN_AGGREGATE_WORKER_CPU_CORES'], check_limits['items']['VIYA_MIN_AGGREGATE_WORKER_MEMORY']) + except KeyError as e: + print() + print(viya_messages.EXCEPTION_MESSAGE.format(e) + + ' Check for missing key/value in the viya_deployment_settings.ini') + print() + sys.exit(viya_messages.RUNTIME_ERROR_RC_) + # gather the details for the report try: print() diff --git a/pre_install_report/test/test_pre_install_report.py b/pre_install_report/test/test_pre_install_report.py index 415509d..0607515 100644 --- a/pre_install_report/test/test_pre_install_report.py +++ b/pre_install_report/test/test_pre_install_report.py @@ -38,7 +38,7 @@ _NAMESPACE_NOT_FOUND_RC_ = 6 _RUNTIME_ERROR_RC_ = 7 -viya_kubelet_version_min = 'v1.14.0' +viya_k8s_version_min = '1.14' viya_kubenetes_version = "1.22.4" viya_min_aggregate_worker_CPU_cores = '12' viya_min_aggregate_worker_memory = '56G' @@ -50,7 +50,7 @@ def test_get_storage_classes_json(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -69,7 +69,7 @@ def test_get_storage_classes_json(): def test_read_cluster_info_output(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) cluster_info = "Kubernetes master is running at https://0.0.0.0:6443\n" + \ @@ -81,7 +81,7 @@ def test_read_cluster_info_output(): def test_delete_temp_file(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -96,7 +96,7 @@ def test_delete_temp_file(): def test_get_master_nodes_json(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -110,7 +110,7 @@ def test_get_master_nodes_json(): def test_ranchersingle_get_master_nodes_json(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -129,7 +129,7 @@ def test_ranchersingle_get_master_nodes_json(): def test_ranchermulti_get_master_nodes_json(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -146,7 +146,7 @@ def test_ranchermulti_get_master_nodes_json(): def test_get_nested_nodes_info(): viya_min_aggregate_worker_CPU_cores = '20' - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -180,17 +180,17 @@ def test_get_nested_nodes_info(): total_aggregate_memoryG = vpc.get_calculated_aggregate_memory() # quantity_("62.3276481628418 Gi").to('G') assert str(round(total_aggregate_memoryG.to("G"), 2)) == '67.13 G' assert str(round(total_aggregate_memoryG.to("Gi"), 2)) == '62.52 Gi' - assert global_data[4]['aggregate_kubelet_failures'] in 'Check Kubelet Version on nodes. Issues Found: 3.' + assert global_data[4]['aggregate_k8s_failures'] in 'Check K8s Version on nodes. Issues Found: 3.' assert global_data[6]['k8sVersion'] in '1.16.1' template_render(global_data, configs_data, storage_data, 'nested_nodes_info.html') def test_get_nested_millicores_nodes_info(): - viya_kubelet_version_min = 'v1.14.0' + viya_k8s_version_min = '1.14' viya_min_aggregate_worker_CPU_cores = '20' viya_min_aggregate_worker_memory = '156G' - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -221,14 +221,14 @@ def test_get_nested_millicores_nodes_info(): ' Issues Found: 1' total_calc_memoryG = vpc.get_calculated_aggregate_memory() assert str(round(total_calc_memoryG.to("G"), 2)) == '68.16 G' # 68.16113098752 - assert global_data[4]['aggregate_kubelet_failures'] in 'Check Kubelet Version on nodes. Issues Found: 3.' + assert global_data[4]['aggregate_k8s_failures'] in 'Check K8s Version on nodes. Issues Found: 3.' assert global_data[6]['k8sVersion'] in '1.13.1' template_render(global_data, configs_data, storage_data, 'nested_millicores_nodes_info.html') def test_ranchersingle_get_nested_nodes_info(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) quantity_ = register_pint() @@ -254,13 +254,13 @@ def test_ranchersingle_get_nested_nodes_info(): ' Issues Found: 0' total_calc_memoryG = vpc.get_calculated_aggregate_memory() assert str(round(total_calc_memoryG.to("G"), 2)) == '67.39 G' - assert global_data[4]['aggregate_kubelet_failures'] in 'Check Kubelet Version on nodes. Issues Found: 0' + assert global_data[4]['aggregate_k8s_failures'] in 'Check Kubelet Version on nodes. Issues Found: 0' template_render(global_data, configs_data, storage_data, 'ranchersingle_nested_nodes_info.html') def test_ranchermulti_get_nested_nodes_info(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) quantity_ = register_pint() @@ -285,13 +285,13 @@ def test_ranchermulti_get_nested_nodes_info(): assert global_data[3]['aggregate_memory_failures'] in 'Expected: 56G, Calculated: 336.94 G,' \ ' Memory within Range,' \ ' Issues Found: 0' - assert global_data[4]['aggregate_kubelet_failures'] in 'Check Kubelet Version on nodes. Issues Found: 0' + assert global_data[4]['aggregate_k8s_failures'] in 'Check Kubelet Version on nodes. Issues Found: 0' template_render(global_data, configs_data, storage_data, 'ranchermulti_nested_nodes_info.html') def test_get_no_config_info(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -332,7 +332,7 @@ def test_check_route_host_port(): def test_get_config_info(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -365,7 +365,7 @@ def test_get_config_info(): def test_ranchersingle_test_get_config_info(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -389,7 +389,7 @@ def test_ranchersingle_test_get_config_info(): def test_ranchermulti_test_get_config_info(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -416,7 +416,7 @@ def test_ranchermulti_test_get_config_info(): def test_azure_terrform_multi_nodes_info(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -441,14 +441,14 @@ def test_azure_terrform_multi_nodes_info(): assert global_data[3]['aggregate_memory_failures'] in 'Expected: 56G, Calculated: 168.54 G,' \ ' Memory within Range,' \ ' Issues Found: 0' - assert global_data[4]['aggregate_kubelet_failures'] in 'Check Kubelet Version on nodes. Issues Found: 0' + assert global_data[4]['aggregate_k8s_failures'] in 'Check Kubelet Version on nodes. Issues Found: 0' template_render(global_data, configs_data, storage_data, 'azure_terrform_multi_nodes_info.html') def test_azure_multi_get_nested_nodes_info(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -476,14 +476,14 @@ def test_azure_multi_get_nested_nodes_info(): assert global_data[3]['aggregate_memory_failures'] in 'Expected: 56G, Calculated: 117.92 G,' \ ' Memory within Range,' \ ' Issues Found: 0' - assert global_data[4]['aggregate_kubelet_failures'] in '0, Check Kubelet Version on nodes.' + assert global_data[4]['aggregate_k8s_failures'] in '0, Check Kubelet Version on nodes.' template_render(global_data, configs_data, storage_data, 'azure_multi_nested_nodes_info.html') def test_azure_worker_nodes(): - viya_kubelet_version_min = 'v1.17.0' - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + viya_k8s_version_min = '1.17' + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -509,8 +509,8 @@ def test_azure_worker_nodes(): assert global_data[3]['aggregate_memory_failures'] in 'Expected: 56G, Calculated: 802.28 G,' \ ' Memory within Range,' \ ' Issues Found: 0' - assert global_data[4]['aggregate_kubelet_failures'] in ' Check Node(s). All Nodes NOT in Ready Status. ' \ - 'Issues Found: ' + str(issues_found) + assert global_data[4]['aggregate_k8s_failures'] in ' Check Node(s). All Nodes NOT in Ready Status. ' \ + 'Issues Found: ' + str(issues_found) assert global_data[6]['k8sVersion'] in '1.22.4' template_render(global_data, configs_data, storage_data, 'azure_nodes_no_master.html') @@ -539,12 +539,12 @@ def register_pint(): return quantity_ -def createViyaPreInstallCheck(viya_kubelet_version_min, +def createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory): sas_pre_check_report: ViyaPreInstallCheck = ViyaPreInstallCheck(sas_logger, - viya_kubelet_version_min, + viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) sas_pre_check_report.set_k8s_version(viya_kubenetes_version) @@ -554,7 +554,7 @@ def createViyaPreInstallCheck(viya_kubelet_version_min, def test_get_calculated_aggregate_memory(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -592,7 +592,7 @@ def test_kubconfig_file(): def test_validated_k8s_server_version(): - vpc = createViyaPreInstallCheck(viya_kubelet_version_min, + vpc = createViyaPreInstallCheck(viya_k8s_version_min, viya_min_aggregate_worker_CPU_cores, viya_min_aggregate_worker_memory) @@ -658,9 +658,10 @@ def test_get_k8s_version(): assert (curr_version in semantic_version.SimpleSpec('<1.20')) assert (curr_version in semantic_version.SimpleSpec('==1.19')) - # current version is less then 1.19 + # current version is less than 1.19 curr_version = semantic_version.Version(str(version_string)) - assert (curr_version in semantic_version.SimpleSpec(viya_constants.MIN_K8S_SERVER_VERSION)) + k8s_version = '<' + version_string2 + assert (curr_version in semantic_version.SimpleSpec(k8s_version)) def test_check_permissions(): diff --git a/pre_install_report/viya_deployment_settings.ini b/pre_install_report/viya_deployment_settings.ini index 4d2a972..ee86cf9 100644 --- a/pre_install_report/viya_deployment_settings.ini +++ b/pre_install_report/viya_deployment_settings.ini @@ -29,6 +29,5 @@ VIYA_MIN_AGGREGATE_WORKER_MEMORY=448G # TOTAL CPU of all worker Nodes in millicores. Sum of the vCPUs on all active nodes required to deploy an offering # Minimum allowed value = '.001'. VIYA_MIN_AGGREGATE_WORKER_CPU_CORES=56 -# Supported versions of Kubelet Version -# Expected format: vxx.xx.0 -VIYA_KUBELET_VERSION_MIN=v1.19.0 \ No newline at end of file +# Minimum Kubernetes Server Version supported. Format: major.minor +VIYA_K8S_VERSION_MIN=1.21 From 43329fca1df3706fffffe4510872b6706c97c30d Mon Sep 17 00:00:00 2001 From: Latha Sivakumar Date: Tue, 8 Nov 2022 10:33:46 -0500 Subject: [PATCH 13/20] (#184) K8s minimum version check --- pre_install_report/pre_install_report.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pre_install_report/pre_install_report.py b/pre_install_report/pre_install_report.py index 2bbc752..470d365 100644 --- a/pre_install_report/pre_install_report.py +++ b/pre_install_report/pre_install_report.py @@ -254,9 +254,9 @@ def main(argv): with LRPIndicator(enter_message="Gathering facts"): try: sas_pre_check_report: ViyaPreInstallCheck = \ - ViyaPreInstallCheck(sas_logger, check_limits['items']['VIYA_K8S_VERSION_MIN'], - check_limits['items']['VIYA_MIN_AGGREGATE_WORKER_CPU_CORES'], - check_limits['items']['VIYA_MIN_AGGREGATE_WORKER_MEMORY']) + ViyaPreInstallCheck(sas_logger, check_limits['items']['VIYA_K8S_VERSION_MIN'], + check_limits['items']['VIYA_MIN_AGGREGATE_WORKER_CPU_CORES'], + check_limits['items']['VIYA_MIN_AGGREGATE_WORKER_MEMORY']) except KeyError as e: print() print(viya_messages.EXCEPTION_MESSAGE.format(e) + From 52e0af4634060504ce9ee013fe7b845761b68827 Mon Sep 17 00:00:00 2001 From: Kevin Lingle <36995745+kevinlinglesas@users.noreply.github.com> Date: Tue, 8 Nov 2022 12:20:30 -0500 Subject: [PATCH 14/20] min k8s version check mentioned in Overview --- pre_install_report/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index 86fcb85..e6cd02d 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -10,7 +10,7 @@ Kubernetes may orchestrate once Viya is deployed. The report and the informatio be considered a snapshot in time. The Kubernetes cluster for a SAS Viya deployment must meet the requirements documented in [SAS® Viya® Operations](https://go.documentation.sas.com/doc/en/itopscdc/default/itopssr/titlepage.htm) -Ensure that the Kubernetes version is within the documented range for the selected cloud provider. +Ensure that the Kubernetes version is within the documented range for the selected cloud provider. If the Kubernetes server version is below a current default minimum, a warning will be included in the report. ### Memory and vCPU Check From bcda53b4878785f18f1376983ee3c04ea4a072f6 Mon Sep 17 00:00:00 2001 From: Kevin Lingle <36995745+kevinlinglesas@users.noreply.github.com> Date: Tue, 8 Nov 2022 12:35:50 -0500 Subject: [PATCH 15/20] Adding to the Modify Settings section --- pre_install_report/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index e6cd02d..984f6fe 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -146,6 +146,10 @@ web-viewable, HTML format. You can modify the /viya4-ark/pre_install_report/viya_deployment_settings.ini file to alter the minimum and aggregate settings for CPU and memory on nodes. For more information, see the details in the file. +If you modify the VIYA_K8S_VERSION_MIN to a version less than the minimum Kubernetes version supported by this +release of the report tool, you are operatiing outside the supported capabilities of the report tool. It is better +to use a release of Viya 4 ARK tools matching the required minimum you may be working with. + ## Known Issues The following issue may impact the performance and expected results of this tool. From ff8b13248b80daa25f7f09d5dfff8b580edcd480 Mon Sep 17 00:00:00 2001 From: Kevin Lingle <36995745+kevinlinglesas@users.noreply.github.com> Date: Tue, 8 Nov 2022 12:40:04 -0500 Subject: [PATCH 16/20] Updated copyright --- pre_install_report/viya_deployment_settings.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre_install_report/viya_deployment_settings.ini b/pre_install_report/viya_deployment_settings.ini index ee86cf9..7e10989 100644 --- a/pre_install_report/viya_deployment_settings.ini +++ b/pre_install_report/viya_deployment_settings.ini @@ -4,7 +4,7 @@ # ### Author: SAS Institute Inc. ### # ################################################################### # ### -# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA. ### +# Copyright (c) 2021-2022, SAS Institute Inc., Cary, NC, USA. ### # All Rights Reserved. ### # SPDX-License-Identifier: Apache-2.0 ### # ### From 5e1cbaffe2d79ee71f295f6cfe2446be230d01d4 Mon Sep 17 00:00:00 2001 From: Kevin Lingle <36995745+kevinlinglesas@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:30:29 -0500 Subject: [PATCH 17/20] Update from review --- pre_install_report/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index 984f6fe..d5738ea 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -10,7 +10,8 @@ Kubernetes may orchestrate once Viya is deployed. The report and the informatio be considered a snapshot in time. The Kubernetes cluster for a SAS Viya deployment must meet the requirements documented in [SAS® Viya® Operations](https://go.documentation.sas.com/doc/en/itopscdc/default/itopssr/titlepage.htm) -Ensure that the Kubernetes version is within the documented range for the selected cloud provider. If the Kubernetes server version is below a current default minimum, a warning will be included in the report. +Ensure that the Kubernetes version is within the documented range for the selected cloud provider. +If the Kubernetes server version is below the default minimum, a warning will be included in the report. ### Memory and vCPU Check @@ -147,7 +148,7 @@ You can modify the /viya4-ark/pre_install_report/viya_deploym minimum and aggregate settings for CPU and memory on nodes. For more information, see the details in the file. If you modify the VIYA_K8S_VERSION_MIN to a version less than the minimum Kubernetes version supported by this -release of the report tool, you are operatiing outside the supported capabilities of the report tool. It is better +release of the report tool, you are operating outside the supported capabilities of the report tool. It is better to use a release of Viya 4 ARK tools matching the required minimum you may be working with. ## Known Issues From 91c512589d668a33b83827de3486616c724f9963 Mon Sep 17 00:00:00 2001 From: Kevin Lingle <36995745+kevinlinglesas@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:35:34 -0500 Subject: [PATCH 18/20] Update from review comments --- pre_install_report/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index d5738ea..422f64f 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -149,7 +149,7 @@ minimum and aggregate settings for CPU and memory on nodes. For more information If you modify the VIYA_K8S_VERSION_MIN to a version less than the minimum Kubernetes version supported by this release of the report tool, you are operating outside the supported capabilities of the report tool. It is better -to use a release of Viya 4 ARK tools matching the required minimum you may be working with. +to use a release of Viya 4 ARK tools matching the required minimum you are working with. ## Known Issues @@ -171,4 +171,4 @@ They will look similar to the resources shown below: kubectl -n delete replicaset.apps/hello-world-6665cf748b kubectl -n delete pod/hello-world-6665cf748b-5x2jq kubectl -n delete pod/hello-world-6665cf748b-tkq79 -``` +``` From 9ed0fc6af05ad25943ba257499634e35a3b34d16 Mon Sep 17 00:00:00 2001 From: Kevin Lingle <36995745+kevinlinglesas@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:38:38 -0500 Subject: [PATCH 19/20] Update from review comments --- pre_install_report/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pre_install_report/README.md b/pre_install_report/README.md index 422f64f..c191d85 100644 --- a/pre_install_report/README.md +++ b/pre_install_report/README.md @@ -148,8 +148,8 @@ You can modify the /viya4-ark/pre_install_report/viya_deploym minimum and aggregate settings for CPU and memory on nodes. For more information, see the details in the file. If you modify the VIYA_K8S_VERSION_MIN to a version less than the minimum Kubernetes version supported by this -release of the report tool, you are operating outside the supported capabilities of the report tool. It is better -to use a release of Viya 4 ARK tools matching the required minimum you are working with. +release of the report tool, you are operating outside the supported capabilities of the report tool. SAS recommends +using a release of Viya 4 ARK tools that matches the required minimum you are working with. ## Known Issues From 6cea51460363798743ee60d1d291bcda4c1abaad Mon Sep 17 00:00:00 2001 From: Latha Sivakumar Date: Thu, 10 Nov 2022 14:17:24 -0500 Subject: [PATCH 20/20] (#184) K8s minimum version check --- pre_install_report/library/pre_install_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre_install_report/library/pre_install_check.py b/pre_install_report/library/pre_install_check.py index 5489956..56d284e 100644 --- a/pre_install_report/library/pre_install_check.py +++ b/pre_install_report/library/pre_install_check.py @@ -77,7 +77,7 @@ def __init__(self, sas_logger: ViyaARKLogger, viya_k8s_version_min, def _parse_release_info(self, release_info): """ - This method checks that the format of the VIYA_K8s_VERSION_MIN specified in the + This method checks that the format of the VIYA_K8S_VERSION_MIN specified in the user modifiable ini file is valid. :param release_info: The minimum K8s version loaded from ini file.