update ats tests

.ats/main.yaml

@@ -1,4 +1,11 @@
-skip-steps: smoke
+app-tests-app-config-file: tests/test-values.yaml
+app-tests-deploy-namespace: monitoring
 
-functional-tests-cluster-type: kind
+smoke-tests-cluster-type: kind
+
+upgrade-tests-cluster-type: kind
+upgrade-tests-app-catalog-url: https://giantswarm.github.io/giantswarm-catalog
+upgrade-tests-app-config-file: tests/test-values.yaml
+
+skip-steps: [functional]
 

.circleci/config.yml

@@ -31,6 +31,16 @@ workflows:
           filters:
             tags:
               only: /^v.*/
+
+      - architect/run-tests-with-ats:
+          name: run-chart-tests-with-ats
+          filters:
+            # Do not trigger the job on merge to master.
+            branches:
+              ignore:
+                - master
+          requires:
+            - "app-catalog"
 
       - architect/push-to-app-collection:
           context: "architect"
@@ -139,12 +149,3 @@ workflows:
               ignore: /.*/
             tags:
               only: /^v.*/
-#      - architect/run-tests-with-ats:
-#          name: execute chart tests
-#          filters:
-#            # Do not trigger the job on merge to master.
-#            branches:
-#              ignore:
-#                - master
-#          requires:
-#            - "package and push grafana chart"

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Update tests for ats usage.
+
 ## [2.13.0] - 2024-07-15
 
 ### Changed

tests/ats/Pipfile

@@ -4,11 +4,11 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-pytest-helm-charts = ">=0.3.1"
-pytest = ">=6.1.2"
-pytest-rerunfailures = "*"
-
-[dev-packages]
+pytest-helm-charts = ">=1.0.2"
+pytest = ">=6.2.5"
+pykube-ng = ">=22.1.0"
+pytest-rerunfailures = "~=14.0"
+requests = "~=2.32.1"
 
 [requires]
-python_version = "3.8"
+python_version = "3.9"

tests/ats/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+markers =
+    smoke
+    functional
+    upgrade

tests/ats/test_grafana.py

@@ -1,16 +1,74 @@
-from typing import cast
+import logging
+import requests
+import time
+from contextlib import contextmanager
+from functools import partial
+from pathlib import Path
+from typing import Dict, List, Optional
 
 import pykube
 import pytest
 from pytest_helm_charts.clusters import Cluster
+from pytest_helm_charts.k8s.deployment import wait_for_deployments_to_run
 
+logger = logging.getLogger(__name__)
 
-# when grafana pod is up, it might be still starting and returning 503
-@pytest.mark.flaky(reruns=10, reruns_delay=10)
-@pytest.mark.functional
-def test_grafana_login_page_available(kube_cluster: Cluster) -> None:
-    srv = cast(pykube.Service, pykube.Service.objects(kube_cluster.kube_client).get_or_none(name="grafana"))
-    if srv is None:
-        raise ValueError("'grafana service not found in the 'default' namespace")
-    login_page_res = srv.proxy_http_get("/login")
-    assert login_page_res.ok
+namespace_name = "monitoring"
+deployment_name= "grafana"
+
+timeout: int = 560
+
+@pytest.mark.smoke
+def test_api_working(kube_cluster: Cluster) -> None:
+    """Very minimalistic example of using the [kube_cluster](pytest_helm_charts.fixtures.kube_cluster)
+    fixture to get an instance of [Cluster](pytest_helm_charts.clusters.Cluster) under test
+    and access its [kube_client](pytest_helm_charts.clusters.Cluster.kube_client) property
+    to get access to Kubernetes API of cluster under test.
+    Please refer to [pykube](https://pykube.readthedocs.io/en/latest/api/pykube.html) to get docs
+    for [HTTPClient](https://pykube.readthedocs.io/en/latest/api/pykube.html#pykube.http.HTTPClient).
+    """
+    assert kube_cluster.kube_client is not None
+    assert len(pykube.Node.objects(kube_cluster.kube_client)) >= 1
+
+# scope "module" means this is run only once, for the first test case requesting! It might be tricky
+# if you want to assert this multiple times
+@pytest.fixture(scope="module")
+def ic_deployment(request, kube_cluster: Cluster) -> List[pykube.Deployment]:
+    logger.info("Waiting for grafana deployment..")
+
+    deployment_ready = wait_for_ic_deployment(kube_cluster)
+
+    logger.info("grafana deployment looks satisfied..")
+
+    return deployment_ready
+
+def wait_for_ic_deployment(kube_cluster: Cluster) -> List[pykube.Deployment]:
+    deployments = wait_for_deployments_to_run(
+        kube_cluster.kube_client,
+        [deployment_name],
+        namespace_name,
+        timeout,
+    )
+    return deployments
+
+
+@pytest.fixture(scope="module")
+def pods(kube_cluster: Cluster) -> List[pykube.Pod]:
+    pods = pykube.Pod.objects(kube_cluster.kube_client)
+
+    pods = pods.filter(namespace=namespace_name, selector={
+                       'app.kubernetes.io/name': 'grafana', 'app.kubernetes.io/instance': 'grafana'})
+
+    return pods
+
+# when we start the tests on circleci, we have to wait for pods to be available, hence
+# this additional delay and retries
+
+
+@pytest.mark.smoke
+@pytest.mark.upgrade
+@pytest.mark.flaky(reruns=5, reruns_delay=10)
+def test_pods_available(kube_cluster: Cluster, ic_deployment: List[pykube.Deployment]):
+    for s in ic_deployment:
+        assert int(s.obj["status"]["readyReplicas"]) == int(
+            s.obj["spec"]["replicas"])

tests/test-values.yaml

@@ -0,0 +1,113 @@
+global:
+  imageRegistry: gsoci.azurecr.io
+grafana:
+  dashboardProviders:
+    dashboardproviders.yaml:
+      apiVersion: 1
+      providers:
+      - allowUiUpdates: false
+        disableDeletion: true
+        editable: false
+        folder: Giant Swarm private dashboards
+        folderUid: GMtRhFeGz
+        name: private
+        options:
+          path: /var/lib/grafana/dashboards/private
+        type: file
+        updateIntervalSeconds: 60
+      - allowUiUpdates: false
+        disableDeletion: true
+        editable: false
+        folder: Giant Swarm public dashboards
+        folderUid: b6utHHg7z
+        name: public
+        options:
+          path: /var/lib/grafana/dashboards/public
+        type: file
+        updateIntervalSeconds: 60
+      - allowUiUpdates: false
+        disableDeletion: true
+        editable: false
+        folder: Your dashboards
+        folderUid: customerDashboards
+        name: customer-provided
+        options:
+          path: /var/lib/grafana/dashboards/customer
+        type: file
+        updateIntervalSeconds: 60
+  grafana.ini:
+    analytics:
+      check_for_updates: false
+      feedback_links_enabled: false
+      reporting_enabled: false
+    auth:
+      disable_login_form: true
+      disable_signout_menu: true
+    auth.basic:
+      enabled: true
+    auth.generic_oauth:
+      allow_assign_grafana_admin: true
+      allow_sign_up: true
+      api_url: https://dex.grizzly.gaws.gigantic.io/userinfo
+      auth_url: https://dex.grizzly.gaws.gigantic.io/auth
+      auto_login: true
+      client_id: 2qRgNb5qQBk3QqWxI1WLgB5zTQ1M4eB+
+      enabled: true
+      name: grafana-dex
+      role_attribute_path: contains(groups, 'giantswarm-ad:giantswarm-admins')  &&
+        'GrafanaAdmin' || contains(groups, 'customer:giantswarm:Employees') && 'Viewer'
+        || contains (groups, 'giantswarm-github:giantswarm:giantswarm-admins')  &&
+        'GrafanaAdmin'
+      role_attribute_strict: true
+      scopes: openid profile email groups offline_access
+      token_url: https://dex.grizzly.gaws.gigantic.io/token
+      use_refresh_token: true
+    dashboards:
+      default_home_dashboard_path: /var/lib/grafana/dashboards/public/home/home.json
+    public_dashboards:
+      enabled: false
+    security:
+      cookie_secure: true
+      disable_gravatar: true
+    server:
+      protocol: http
+      root_url: https://grafana.grizzly.gaws.gigantic.io
+    users:
+      allow_org_create: false
+      allow_sign_up: false
+      viewers_can_edit: true
+  image:
+    repository: giantswarm/grafana
+  ingress:
+    annotations:
+      cert-manager.io/cluster-issuer: letsencrypt-giantswarm
+      kubernetes.io/tls-acme: "true"
+    enabled: true
+    hosts:
+    - grafana.grizzly.gaws.gigantic.io
+    ingressClassName: nginx
+    tls:
+    - hosts:
+      - grafana.grizzly.gaws.gigantic.io
+      secretName: grafana-tls
+  rbac:
+    pspEnabled: false
+  serviceMonitor:
+    enabled: true
+  sidecar:
+    dashboards:
+      SCProvider: false
+      enabled: true
+      folder: /var/lib/grafana/dashboards
+      label: app.giantswarm.io/kind
+      labelValue: dashboard
+      searchNamespace: ALL
+    datasources:
+      enabled: true
+      label: app.giantswarm.io/kind
+      labelValue: datasource
+      resource: both
+      searchNamespace: ALL
+    image:
+      repository: giantswarm/k8s-sidecar
+