Merge pull request #160 from sassoftware/develop

Prepare for release 1.7.0

deployment_report/model/static/viya_deployment_report_keys.py

@@ -45,11 +45,13 @@ class Kubernetes(object):
         API_RESOURCES_DICT = "apiResources"
         API_VERSIONS_LIST = "apiVersions"
         CADENCE_INFO = "cadenceInfo"
+        CONFIGMAPS_DICT = "configMaps"
         DB_INFO = "dbInfo"
         DISCOVERED_RESOURCE_TYPES_DICT = "discoveredResourceTypes"
         INGRESS_CTRL = "ingressController"
         NAMESPACE = "namespace"
         NODES_DICT = "nodes"
+        SECRETS_DICT = "secrets"
         VERSIONS_DICT = "versions"
 
     class ResourceDetails(object):

deployment_report/model/test/test_viya_deployment_report.py

@@ -89,15 +89,17 @@ def test_get_kubernetes_details(report: ViyaDeploymentReport) -> None:
     kube_details: Dict = report.get_kubernetes_details()
 
     # check for all expected entries
-    assert len(kube_details) == 9
+    assert len(kube_details) == 11
     assert ReportKeys.Kubernetes.API_RESOURCES_DICT in kube_details
     assert ReportKeys.Kubernetes.API_VERSIONS_LIST in kube_details
     assert ReportKeys.Kubernetes.CADENCE_INFO in kube_details
+    assert ReportKeys.Kubernetes.CONFIGMAPS_DICT in kube_details
     assert ReportKeys.Kubernetes.DB_INFO in kube_details
     assert ReportKeys.Kubernetes.DISCOVERED_RESOURCE_TYPES_DICT in kube_details
     assert ReportKeys.Kubernetes.INGRESS_CTRL in kube_details
     assert ReportKeys.Kubernetes.NAMESPACE in kube_details
     assert ReportKeys.Kubernetes.NODES_DICT in kube_details
+    assert ReportKeys.Kubernetes.SECRETS_DICT in kube_details
     assert ReportKeys.Kubernetes.VERSIONS_DICT in kube_details
 
 
@@ -121,7 +123,7 @@ def test_get_api_resources() -> None:
     api_resources: Dict = report.get_api_resources()
 
     # check for expected attributes
-    assert len(api_resources) == 17
+    assert len(api_resources) == 18
 
     assert ResourceTypeValues.CONTOUR_HTTP_PROXIES in api_resources
     assert ResourceTypeValues.ISTIO_VIRTUAL_SERVICES
@@ -133,6 +135,7 @@ def test_get_api_resources() -> None:
     assert ResourceTypeValues.K8S_CORE_CONFIG_MAPS in api_resources
     assert ResourceTypeValues.K8S_CORE_NODES in api_resources
     assert ResourceTypeValues.K8S_CORE_PODS in api_resources
+    assert ResourceTypeValues.K8S_CORE_SECRETS in api_resources
     assert ResourceTypeValues.K8S_CORE_SERVICES in api_resources
     assert ResourceTypeValues.K8S_EXTENSIONS_INGRESSES in api_resources
     assert ResourceTypeValues.K8S_METRICS_NODES in api_resources
@@ -192,7 +195,7 @@ def test_get_discovered_resources(report: ViyaDeploymentReport) -> None:
     discovered_resources: Dict = report.get_discovered_resources()
 
     # check for expected attributes
-    assert len(discovered_resources) == 15
+    assert len(discovered_resources) == 16
     assert ResourceTypeValues.CONTOUR_HTTP_PROXIES in discovered_resources
     assert ResourceTypeValues.ISTIO_VIRTUAL_SERVICES in discovered_resources
     assert ResourceTypeValues.K8S_APPS_DEPLOYMENTS in discovered_resources
@@ -203,6 +206,7 @@ def test_get_discovered_resources(report: ViyaDeploymentReport) -> None:
     assert ResourceTypeValues.K8S_CORE_CONFIG_MAPS in discovered_resources
     assert ResourceTypeValues.K8S_CORE_NODES in discovered_resources
     assert ResourceTypeValues.K8S_CORE_PODS in discovered_resources
+    assert ResourceTypeValues.K8S_CORE_SECRETS in discovered_resources
     assert ResourceTypeValues.K8S_CORE_SERVICES in discovered_resources
     assert ResourceTypeValues.K8S_EXTENSIONS_INGRESSES in discovered_resources
     assert ResourceTypeValues.K8S_NETWORKING_INGRESSES in discovered_resources

deployment_report/model/utils/config_util.py

@@ -46,6 +46,7 @@
 _INTERNAL_DB_ = "Internal"
 _UNSUPPORTED_DB_ = "Unsupported"
 _CRUNCH_DB_ = "crunch"
+_CRUNCHDATA_DB_ = "sas-crunchy-data-"
 _ALT_DB_ = "sas.database.alternative.databaseServerName"
 _UNAVAIL_DB_ = "not available"
 
@@ -115,13 +116,10 @@ def get_db_info(resource_cache: Dict) -> Dict:
     if not resource_cache:
         return None
 
-    try:
+    if ResourceTypeValues.SAS_PGCLUSTERS in resource_cache.keys():
         pgclusters: Dict = resource_cache[ResourceTypeValues.SAS_PGCLUSTERS][ITEMS_KEY]
         db_dict = _get_db_info_v2(pgclusters)
 
-    except KeyError:
-        pass
-
     if not db_dict:
         try:
             config_maps: Dict = resource_cache[ResourceTypeValues.K8S_CORE_CONFIG_MAPS][ITEMS_KEY]
@@ -230,11 +228,84 @@ def _get_db_info_v2(pgclusters: Dict) -> Dict:
                             Keys.DatabaseDetails.DBCONN: _UNAVAIL_DB_
                         }
             except KeyError:
-                dbs = {Keys.DatabaseDetails.DBTYPE: _UNSUPPORTED_DB_}
+                continue
 
             if dbs:
+                # convert db_name to be aligned
+                if _CRUNCHDATA_DB_ in db_name:
+                    db_name = db_name.replace(_CRUNCHDATA_DB_, "")
+
                 db_dict[db_name] = dbs
         except KeyError:
             continue
 
     return db_dict
+
+
+def get_configmaps_info(resource_cache: Dict) -> Optional[Dict]:
+    """
+    Returns the configmaps of the targeted SAS deployment.
+
+    :param resource_cache: The dictionary of resources gathered from the k8s deployment.
+
+    :return: A dictionary representing the configmaps information of the targeted SAS deployment.
+    """
+    configmaps_dict: Dict = dict()
+    try:
+        if not resource_cache:
+            return configmaps_dict
+
+        configmaps: Dict = resource_cache[ResourceTypeValues.K8S_CORE_CONFIG_MAPS][ITEMS_KEY]
+
+        for configmap_name, configmap_details in configmaps.items():
+
+            configmap: KubernetesResource = configmap_details[Keys.ResourceDetails.RESOURCE_DEFINITION]
+            # get the ConfigMap data
+            data: Optional[Dict] = configmap.get_data()
+
+            if data:
+                configmaps_dict[configmap_name] = data
+
+        return configmaps_dict
+
+    except KeyError:
+        return configmaps_dict
+
+
+def get_secrets_info(resource_cache: Dict, secretsnames_dict: Dict) -> Optional[Dict]:
+    """
+    Returns the secrets of the targeted SAS deployment.
+
+    :param resource_cache: The dictionary of resources gathered from the k8s deployment.
+    :param secretsnames_dict: The dictionary to store the secrets
+
+    :return: A dictionary representing the secrets by kind of the targeted SAS deployment.
+    """
+    secrets_dict: Dict = dict()
+    try:
+        if not resource_cache:
+            return secrets_dict
+
+        secrets: Dict = resource_cache[ResourceTypeValues.K8S_CORE_SECRETS][ITEMS_KEY]
+
+        for name, details in secrets.items():
+            secret: KubernetesResource = details[Keys.ResourceDetails.RESOURCE_DEFINITION]
+            data: Optional[Dict] = secret.get_data()
+
+            if data:
+                type: Optional[Text] = secret.get_type()
+
+                if type in secrets_dict.keys():
+                    if name not in secrets_dict[type].keys():
+                        secrets_dict[type][name] = {"name": list(data.keys())}
+                else:
+                    secrets_dict[type]: Dict = dict()
+                    secrets_dict[type][name]: Dict = dict()
+                    secrets_dict[type][name] = {"name": list(data.keys())}
+
+                secretsnames_dict[name] = type
+
+        return secrets_dict
+
+    except KeyError:
+        return secrets_dict

deployment_report/model/utils/resource_util.py

@@ -114,6 +114,10 @@ def cache_resources(resource_type: Text, kubectl: KubectlInterface, resource_cac
         # create a key under which the resource definition will be stored: KubernetesResource
         resource_details[Keys.ResourceDetails.RESOURCE_DEFINITION]: KubernetesResource = resource
 
+        # skip ownerReferences for secret and configmap
+        if resource_type == "configmaps" or resource_type == "secrets":
+            continue
+
         # see if this resource defines any 'ownerReferences'
         owner_references: Optional[List[Dict]] = resource.get_metadata_value(KubernetesResourceKeys.OWNER_REFERENCES)
 

deployment_report/model/utils/test/test_config_util.py

@@ -86,3 +86,28 @@ def test_get_db_info_none():
     db_dict: Dict = config_util.get_db_info(None)
 
     assert db_dict is None
+
+
+@pytest.mark.usefixtures(conftest.NO_INGRESS_SIMULATION_FIXTURE)
+def test_get_configmaps_info(no_ingress_simulation_fixture: conftest.DSA):
+    """
+    This test verifies that the provided configmap data is returned when values is passed to get_configmaps_info().
+    """
+    # get the resource cache
+    resource_cache: Dict = no_ingress_simulation_fixture.resource_cache()
+
+    # test the util method
+    assert len(config_util.get_configmaps_info(resource_cache)) == 8
+
+
+@pytest.mark.usefixtures(conftest.NO_INGRESS_SIMULATION_FIXTURE)
+def test_get_secrets_info(no_ingress_simulation_fixture: conftest.DSA):
+    """
+    This test verifies that the provided secrets data is returned when values is passed to get_secrets_info().
+    """
+    # get the resource cache
+    resource_cache: Dict = no_ingress_simulation_fixture.resource_cache()
+
+    # test the util method
+    secretsnames_dict: Dict = dict()
+    assert len(config_util.get_secrets_info(resource_cache, secretsnames_dict)) == 2

deployment_report/model/viya_deployment_report.py

@@ -27,6 +27,7 @@
     resource_util
 
 from viya_ark_library.jinja2.sas_jinja2 import Jinja2TemplateRenderer
+from viya_ark_library.k8s.k8s_resource_keys import KubernetesResourceKeys
 from viya_ark_library.k8s.k8s_resource_type_values import KubernetesResourceTypeValues as ResourceTypeValues
 from viya_ark_library.k8s.sas_k8s_errors import KubectlRequestForbiddenError
 from viya_ark_library.k8s.sas_k8s_ingress import SupportedIngress
@@ -46,6 +47,14 @@
 # SAS custom API resource group id
 _SAS_API_GROUP_ID_ = "sas.com"
 
+# Configuration keys
+_ENVFROM_ = "envFrom"
+_PODS_ = "pods"
+_CONFIGMAPREF_ = "configMapRef"
+_SECRETREF_ = "secretRef"
+_REFPC_ = "referenced_pods_containers"
+_REFLINK_ = "ref"
+
 
 class ViyaDeploymentReport(object):
     """
@@ -137,11 +146,13 @@ def _create_resource_cache(kubectl: KubectlInterface, sas_custom_resource_types:
 
         # cache values that are not owned by components but could still be included in the report, if present
         for resource_type in [ResourceTypeValues.K8S_CORE_NODES,
-                              ResourceTypeValues.K8S_CORE_CONFIG_MAPS]:
+                              ResourceTypeValues.K8S_CORE_CONFIG_MAPS,
+                              ResourceTypeValues.K8S_CORE_SECRETS]:
             try:
                 ################
                 # Nodes
                 # ConfigMaps
+                # Secrets
                 ################
                 resource_util.cache_resources(resource_type=resource_type,
                                               kubectl=kubectl,
@@ -243,12 +254,18 @@ def gather_details(self, kubectl: KubectlInterface,
                                                            sas_custom_resource_types=sas_custom_resource_types)
 
         #######################################################################
-        # Create a cadence/db information                                     #
+        # Create a cadence/db/configmaps/secrets information                  #
         #######################################################################
         cadence_info: Text = config_util.get_cadence_version(resource_cache=resource_cache)
 
         db_dict: Dict = config_util.get_db_info(resource_cache=resource_cache)
 
+        configmaps_dict: Dict = config_util.get_configmaps_info(resource_cache=resource_cache)
+
+        secretsnames_dict: Dict = dict()
+        secrets_dict: Dict = config_util.get_secrets_info(resource_cache=resource_cache,
+                                                          secretsnames_dict=secretsnames_dict)
+
         #######################################################################
         # Check whether pod resources were found (resources exist)            #
         #######################################################################
@@ -362,6 +379,12 @@ def gather_details(self, kubectl: KubectlInterface,
         # create a key to hold the Viya db information: dict
         k8s_details_dict[Keys.Kubernetes.DB_INFO]: Dict = db_dict
 
+        # create a key to hold the Viya configmaps information: dict
+        k8s_details_dict[Keys.Kubernetes.CONFIGMAPS_DICT]: Dict = configmaps_dict
+
+        # create a key to hold the Viya secrets information: dict
+        k8s_details_dict[Keys.Kubernetes.SECRETS_DICT]: Dict = secrets_dict
+
         # add the availability and count of all discovered resources
         for resource_type, resource_type_details in resource_cache.items():
             # create an entry for the resource
@@ -411,6 +434,10 @@ def gather_details(self, kubectl: KubectlInterface,
                         is_sas_component = \
                             (is_sas_component or
                              resource_details[Keys.ResourceDetails.RESOURCE_DEFINITION].is_sas_resource())
+                        if is_sas_component:
+                            break
+                    if is_sas_component:
+                        break
 
                 # add the component to its appropriate dictionary
                 if is_sas_component:
@@ -432,6 +459,45 @@ def gather_details(self, kubectl: KubectlInterface,
                     # add this to the misc dict, it could not be treated as a SAS component
                     misc_dict[component_name]: Dict = component[ITEMS_KEY]
 
+            # build relationship configmaps to pod containers
+            refcfgmaps_dict: Dict = dict()
+            refsecrets_dict: Dict = dict()
+
+            for s in sas_dict.keys():
+                for mypod in sas_dict[s][_PODS_].keys():
+                    myres = sas_dict[s][_PODS_][mypod][Keys.ResourceDetails.RESOURCE_DEFINITION]
+                    mypodname = myres[KubernetesResourceKeys.METADATA][NAME_KEY]
+                    myspec = myres.get_spec()
+                    for c in myspec[KubernetesResourceKeys.CONTAINERS]:
+                        if _ENVFROM_ in c.keys():
+                            for e in c[_ENVFROM_]:
+                                if _CONFIGMAPREF_ in e.keys():
+                                    try:
+                                        refcfgmaps_dict[e[_CONFIGMAPREF_][NAME_KEY]].append(
+                                            mypodname + "_" + c[NAME_KEY])
+                                    except KeyError:
+                                        refcfgmaps_dict[e[_CONFIGMAPREF_][NAME_KEY]] = [mypodname + "_" + c[NAME_KEY]]
+
+                                elif _SECRETREF_ in e.keys():
+                                    try:
+                                        refsecrets_dict[e[_SECRETREF_][NAME_KEY]].append(mypodname + "_" + c[NAME_KEY])
+                                    except KeyError:
+                                        refsecrets_dict[e[_SECRETREF_][NAME_KEY]] = [mypodname + "_" + c[NAME_KEY]]
+
+            for k in refcfgmaps_dict.keys():
+                try:
+                    configmaps_dict[k][_REFPC_] = refcfgmaps_dict[k]
+                except KeyError:
+                    continue
+
+            # assign ref secrets
+            for r in refsecrets_dict.keys():
+                try:
+                    k = secretsnames_dict[r]
+                    secrets_dict[k][r][_REFLINK_] = refsecrets_dict[r]
+                except KeyError:
+                    continue
+
     def get_kubernetes_details(self) -> Optional[Dict]:
         """
         Returns the details gathered about the target Kubernetes cluster.