Merge pull request #134 from sassoftware/develop

Merge develop branch in preparation for release 1.4.0

deployment_report/model/static/viya_deployment_report_keys.py

@@ -4,7 +4,7 @@
 # ### Author: SAS Institute Inc.                                 ###
 ####################################################################
 #                                                                ###
-# Copyright (c) 2020, SAS Institute Inc., Cary, NC, USA.         ###
+# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA.         ###
 # All Rights Reserved.                                           ###
 # SPDX-License-Identifier: Apache-2.0                            ###
 #                                                                ###
@@ -44,6 +44,7 @@ class Kubernetes(object):
         API_RESOURCES_DICT = "apiResources"
         API_VERSIONS_LIST = "apiVersions"
         CADENCE_INFO = "cadenceInfo"
+        DB_INFO = "dbInfo"
         DISCOVERED_KINDS_DICT = "discoveredKinds"
         INGRESS_CTRL = "ingressController"
         NAMESPACE = "namespace"

deployment_report/model/test/test_viya_deployment_report.py

@@ -4,7 +4,7 @@
 # ### Author: SAS Institute Inc.                                 ###
 ####################################################################
 #                                                                ###
-# Copyright (c) 2020, SAS Institute Inc., Cary, NC, USA.         ###
+# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA.         ###
 # All Rights Reserved.                                           ###
 # SPDX-License-Identifier: Apache-2.0                            ###
 #                                                                ###
@@ -18,10 +18,9 @@
 from deployment_report.model.viya_deployment_report import ViyaDeploymentReport
 from deployment_report.model.static.viya_deployment_report_keys import ITEMS_KEY
 from deployment_report.model.static.viya_deployment_report_keys import ViyaDeploymentReportKeys as ReportKeys
-from deployment_report.model.static.viya_deployment_report_ingress_controller import \
-    ViyaDeploymentReportIngressController as ExpectedIngressController
 
 from viya_ark_library.k8s.sas_k8s_errors import KubectlRequestForbiddenError
+from viya_ark_library.k8s.sas_k8s_ingress import SupportedIngress
 from viya_ark_library.k8s.sas_k8s_objects import KubernetesResource
 from viya_ark_library.k8s.test_impl.sas_kubectl_test import KubectlTest
 
@@ -86,10 +85,11 @@ def test_get_kubernetes_details(report: ViyaDeploymentReport) -> None:
     kube_details: Dict = report.get_kubernetes_details()
 
     # check for all expected entries
-    assert len(kube_details) == 8
+    assert len(kube_details) == 9
     assert ReportKeys.Kubernetes.API_RESOURCES_DICT in kube_details
     assert ReportKeys.Kubernetes.API_VERSIONS_LIST in kube_details
     assert ReportKeys.Kubernetes.CADENCE_INFO in kube_details
+    assert ReportKeys.Kubernetes.DB_INFO in kube_details
     assert ReportKeys.Kubernetes.DISCOVERED_KINDS_DICT in kube_details
     assert ReportKeys.Kubernetes.INGRESS_CTRL in kube_details
     assert ReportKeys.Kubernetes.NAMESPACE in kube_details
@@ -105,25 +105,30 @@ def test_get_kubernetes_details_unpopulated() -> None:
     assert ViyaDeploymentReport().get_kubernetes_details() is None
 
 
-def test_get_api_resources(report: ViyaDeploymentReport) -> None:
+def test_get_api_resources() -> None:
     """
     This test verifies that all the expected api-resources values returned by the KubectlTest implementation are
     present in the "kubernetes.apiResources" dictionary in the completed report.
-
-    :param report: The populated ViyaDeploymentReport returned by the report() fixture.
     """
+    report: ViyaDeploymentReport = ViyaDeploymentReport()
+    report.gather_details(kubectl=KubectlTest(ingress_simulator=KubectlTest.IngressSimulator.ALL_NGINX_USED))
+
     # get the API resources information
     api_resources: Dict = report.get_api_resources()
 
     # check for expected attributes
-    assert len(api_resources) == 12
+    assert len(api_resources) == 16
     assert KubernetesResource.Kinds.CAS_DEPLOYMENT in api_resources
+    assert KubernetesResource.Kinds.CONFIGMAP in api_resources
+    assert KubernetesResource.Kinds.CONTOUR_HTTPPROXY in api_resources
     assert KubernetesResource.Kinds.CRON_JOB in api_resources
     assert KubernetesResource.Kinds.DEPLOYMENT in api_resources
     assert KubernetesResource.Kinds.INGRESS in api_resources
+    assert KubernetesResource.Kinds.ISTIO_VIRTUAL_SERVICE
     assert KubernetesResource.Kinds.JOB in api_resources
     assert KubernetesResource.Kinds.NODE in api_resources
     assert KubernetesResource.Kinds.NODE_METRICS in api_resources
+    assert KubernetesResource.Kinds.OPENSHIFT_ROUTE in api_resources
     assert KubernetesResource.Kinds.POD in api_resources
     assert KubernetesResource.Kinds.POD_METRICS in api_resources
     assert KubernetesResource.Kinds.REPLICA_SET in api_resources
@@ -181,11 +186,13 @@ def test_get_discovered_resources(report: ViyaDeploymentReport) -> None:
     discovered_resources: Dict = report.get_discovered_resources()
 
     # check for expected attributes
-    assert len(discovered_resources) == 11
+    assert len(discovered_resources) == 13
     assert KubernetesResource.Kinds.CAS_DEPLOYMENT in discovered_resources
+    assert KubernetesResource.Kinds.CONTOUR_HTTPPROXY in discovered_resources
     assert KubernetesResource.Kinds.CRON_JOB in discovered_resources
     assert KubernetesResource.Kinds.DEPLOYMENT in discovered_resources
     assert KubernetesResource.Kinds.INGRESS in discovered_resources
+    assert KubernetesResource.Kinds.ISTIO_VIRTUAL_SERVICE in discovered_resources
     assert KubernetesResource.Kinds.JOB in discovered_resources
     assert KubernetesResource.Kinds.NODE in discovered_resources
     assert KubernetesResource.Kinds.POD in discovered_resources
@@ -212,54 +219,15 @@ def test_get_discovered_resources_unpopulated() -> None:
     assert ViyaDeploymentReport().get_discovered_resources() is None
 
 
-def test_get_ingress_controller_nginx_only(report: ViyaDeploymentReport) -> None:
+def test_get_ingress_controller(report: ViyaDeploymentReport) -> None:
     """
     This test verifies that NGINX is returned as the ingress controller when VirtualService objects are not available
     in the api-resources.
 
     :param report: The populated ViyaDeploymentReport returned by the report() fixture.
     """
     # check for expected attributes
-    assert report.get_ingress_controller() == ExpectedIngressController.KUBE_NGINX
-
-
-def test_get_ingress_controller_istio_only() -> None:
-    """
-    This test verifies that ISTIO is returned as the ingress controller when Ingress objects are not available in the
-    api-resources.
-    """
-    # run the report
-    report: ViyaDeploymentReport = ViyaDeploymentReport()
-    report.gather_details(kubectl=KubectlTest(KubectlTest.IngressSimulator.ISTIO_ONLY))
-
-    # check for expected attributes
-    assert report.get_ingress_controller() == ExpectedIngressController.ISTIO
-
-
-def test_get_ingress_controller_both_nginx_used() -> None:
-    """
-    This test verifies that NGINX is returned as the ingress controller when both Ingress and VirtualService resources
-    are available but Ingress is defined for components.
-    """
-    # run the report
-    report: ViyaDeploymentReport = ViyaDeploymentReport()
-    report.gather_details(kubectl=KubectlTest(KubectlTest.IngressSimulator.BOTH_RESOURCES_NGINX_USED))
-
-    # check for expected attributes
-    assert report.get_ingress_controller() is ExpectedIngressController.KUBE_NGINX
-
-
-def test_get_ingress_controller_both_istio_used() -> None:
-    """
-    This test verifies that ISTIO is returned as the ingress controller when both Ingress and VirtualService resources
-    are available but VirtualService is defined for components.
-    """
-    # run the report
-    report: ViyaDeploymentReport = ViyaDeploymentReport()
-    report.gather_details(kubectl=KubectlTest(KubectlTest.IngressSimulator.BOTH_RESOURCES_ISTIO_USED))
-
-    # check for expected attributes
-    assert report.get_ingress_controller() is ExpectedIngressController.ISTIO
+    assert report.get_ingress_controller() == SupportedIngress.Controllers.NGINX
 
 
 def test_get_ingress_controller_none() -> None:
@@ -603,32 +571,3 @@ def test_write_report_unpopulated() -> None:
     # make sure None is returned
     assert data_file is None
     assert html_file is None
-
-
-def test_get_cadence_version(report: ViyaDeploymentReport) -> None:
-    """
-    This test verifies that the provided cadence data is returned when values is passed to get_cadence_version().
-
-    :param report: The populated ViyaDeploymentReport returned by the report() fixture.
-    """
-    # check for expected attributes
-
-    cadence_data = KubectlTest.get_resources(KubectlTest(), "ConfigMaps")
-    cadence_info: Text = None
-
-    for c in cadence_data:
-        cadence_info = report.get_cadence_version(c)
-        if cadence_info:
-            break
-
-    assert cadence_info == KubectlTest.Values.CADENCEINFO
-
-
-def test_get_cadence_version_none() -> None:
-    """
-    This test verifies that a None value is returned for the cadence when the report is unpopulated.
-    """
-
-    # make sure None is returned
-    assert ViyaDeploymentReport().get_sas_component_resources(KubectlTest.Values.CADENCEINFO,
-                                                              KubernetesResource.Kinds.CONFIGMAP) is None

deployment_report/model/utils/component_util.py

@@ -0,0 +1,87 @@
+####################################################################
+# ### component_util.py                                          ###
+####################################################################
+# ### Author: SAS Institute Inc.                                 ###
+####################################################################
+#                                                                ###
+# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA.         ###
+# All Rights Reserved.                                           ###
+# SPDX-License-Identifier: Apache-2.0                            ###
+#                                                                ###
+####################################################################
+from typing import Dict, List, Optional, Text
+
+from deployment_report.model.static.viya_deployment_report_keys import \
+    ITEMS_KEY, \
+    NAME_KEY, \
+    ViyaDeploymentReportKeys as ReportKeys
+
+from viya_ark_library.k8s.sas_k8s_objects import KubernetesResource
+
+
+def aggregate_resources(resource_details: Dict, gathered_resources: Dict, component: Dict):
+    """
+    Aggregates the various resources that comprise a SAS component deployed into the Kubernetes cluster.
+
+    This method is called recursively for each relationship extension to aggregate all related resources.
+
+    :param resource_details: The details of the resource to aggregate into a component.
+    :param gathered_resources: The complete dictionary of resources gathered in the Kubernetes cluster.
+    :param component: The dictionary where the resources for the current component will be compiled.
+    """
+    # set up the component dict
+    if NAME_KEY not in component:
+        component[NAME_KEY]: Text = ""
+
+    if ITEMS_KEY not in component:
+        component[ITEMS_KEY]: Dict = dict()
+
+    # get the relationships extension list for this resource
+    resource_ext: Dict = resource_details[ReportKeys.ResourceDetails.EXT_DICT]
+    resource_relationships: List = resource_ext[ReportKeys.ResourceDetails.Ext.RELATIONSHIPS_LIST]
+
+    # get the resource definition
+    resource: KubernetesResource = resource_details[ReportKeys.ResourceDetails.RESOURCE_DEFINITION]
+
+    # if a SAS component name is defined, use it since this is the most canonical value
+    if resource.get_annotation(KubernetesResource.Keys.ANNOTATION_COMPONENT_NAME) is not None:
+        component[NAME_KEY] = resource.get_annotation(KubernetesResource.Keys.ANNOTATION_COMPONENT_NAME)
+
+    # if a resource of this kind hasn't been added for the component, create the kind key
+    if resource.get_kind() not in component[ITEMS_KEY]:
+        component[ITEMS_KEY][resource.get_kind()]: Dict = dict()
+
+    # add the resource details to its kind dictionary, keyed by its name
+    component[ITEMS_KEY][resource.get_kind()][resource.get_name()]: Dict = resource_details
+
+    # aggregate any resources defined in the relationships extension #
+    for relationship in resource_relationships:
+        # get the name and kind of the related resource #
+        rel_kind: Text = relationship[ReportKeys.ResourceDetails.Ext.Relationship.KIND]
+        rel_name: Text = relationship[ReportKeys.ResourceDetails.Ext.Relationship.NAME]
+
+        # get the details for the related resource
+        try:
+            related_resource_details: Optional[Dict] = gathered_resources[rel_kind][ITEMS_KEY][rel_name]
+        except KeyError:
+            # ignore any failures that may be raised if the resource is transient and not defined
+            # note that the related resource wasn't found
+            related_resource_details: Optional[Dict] = None
+
+        # aggregate the related resource
+        if related_resource_details is not None:
+            try:
+                aggregate_resources(related_resource_details, gathered_resources, component)
+            except RecursionError:
+                # TODO: refactor this error handling; update kubectl get calls to use kind name and group
+                # for now, move on if a recursion error is hit
+                continue
+
+    # if this is the last resource in the chain and the component doesn't have a name determined from an annotation,
+    # set a name based on the available values
+    if not component[NAME_KEY]:
+        component[NAME_KEY]: Text = resource.get_name()
+
+    # if a SAS component name is defined, use it instead
+    if resource.get_annotation(KubernetesResource.Keys.ANNOTATION_COMPONENT_NAME) is not None:
+        component[NAME_KEY] = resource.get_annotation(KubernetesResource.Keys.ANNOTATION_COMPONENT_NAME)

deployment_report/model/utils/config_util.py

@@ -0,0 +1,79 @@
+####################################################################
+# ### config_util.py                                             ###
+####################################################################
+# ### Author: SAS Institute Inc.                                 ###
+####################################################################
+#                                                                ###
+# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA.         ###
+# All Rights Reserved.                                           ###
+# SPDX-License-Identifier: Apache-2.0                            ###
+#                                                                ###
+####################################################################
+from typing import Dict, Optional, Text
+
+from viya_ark_library.k8s.sas_k8s_objects import KubernetesResource
+
+
+def get_cadence_version(config_map: KubernetesResource) -> Optional[Text]:
+    """
+    Returns the cadence version of the targeted SAS deployment.
+
+    :param config_map: The ConfigMap resource to evaluate.
+    :return: A string representing the cadence version of the targeted SAS deployment.
+    """
+    # initialize the return value
+    cadence_info: Optional[Text] = None
+
+    try:
+        # look for ConfigMap with expected name
+        if 'sas-deployment-metadata' in config_map.get_name():
+            # get the ConfigMap data
+            cadence_data: Optional[Dict] = config_map.get_data()
+
+            # build the cadence string
+            cadence_info = (
+                f"{cadence_data['SAS_CADENCE_DISPLAY_NAME']} "
+                f"{cadence_data['SAS_CADENCE_VERSION']} "
+                f"({cadence_data['SAS_CADENCE_RELEASE']})"
+            )
+
+        return cadence_info
+    except KeyError:
+        # if an expected key wasn't defined, return None
+        return None
+
+
+def get_db_info(config_map: KubernetesResource) -> Optional[Dict]:
+    """
+    Returns the db information of the targeted SAS deployment.
+
+    :param config_map: The ConfigMap resource to evaluate.
+    :return: A dict representing the db information of the targeted SAS deployment.
+    """
+    # initialize the return value
+    db_dict: Optional[Dict] = dict()
+
+    try:
+        # make sure the ConfigMap has the expected name
+        if 'sas-postgres-config' in config_map.get_name():
+            # get the ConfigMap data
+            db_data: Optional[Dict] = config_map.get_data()
+
+            # check whether the db configuration is external
+            if db_data['EXTERNAL_DATABASE'] == "false":
+                # return internal config information (all other details will be defined in the component in report)
+                return {"Type": "Internal"}
+
+            # if external, create the dict of all relevant info
+            db_dict = {
+                "Type": "External",
+                "Host": db_data['DATABASE_HOST'],
+                "Port": db_data['DATABASE_PORT'],
+                "Name": db_data['DATABASE_NAME'],
+                "User": db_data['SPRING_DATASOURCE_USERNAME']
+            }
+
+        return db_dict
+    except KeyError:
+        # if an expected key wasn't defined, return None
+        return None