Metric verification job

.zuul.yaml

@@ -50,6 +50,22 @@
     required-projects: *required_projects
     irrelevant-files: *irrelevant_files
 
+- job:
+    name: functional-metric-verification-tests-osp18
+    parent: telemetry-operator-multinode-autoscaling
+    description: |
+      Run the metric verification functional test on osp18.
+    vars:
+      patch_observabilityclient: true
+      cifmw_extras:
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-metric-verification-test.yml"
+    roles:
+      - zuul: github.com/openstack-k8s-operators/ci-framework
+    required-projects: *required_projects
+    irrelevant-files: *irrelevant_files
+
 - job:
     name: feature-verification-tests-noop
     parent: noop
@@ -87,3 +103,4 @@
         - functional-tests-on-osp18
         - functional-logging-tests-osp18
         - functional-graphing-tests-osp18
+        - functional-metric-verification-tests-osp18

ci/run_verify_metrics_osp18.yml

@@ -0,0 +1,28 @@
+---
+- name: Run telemetry tests to verify metrics on osp18
+  hosts:  "{{ cifmw_target_hook_host | default('localhost')  }}"
+  gather_facts: true
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  vars_files:
+    - vars/common.yml
+    - vars/osp18_env.yml
+  tasks:
+    - name: Include vars from the extra_vars files
+      ansible.builtin.include_vars:
+        dir: "{{ cifmw_basedir }}/artifacts/parameters"
+
+    - name: Patch observabilityclient into openstackclient
+      ansible.builtin.shell:
+        cmd: |
+          oc exec openstackclient -- python3 -m ensurepip --upgrade
+          oc exec openstackclient -- python3 -m pip install --upgrade aodhclient
+          oc exec openstackclient -- python3 -m pip install --upgrade python-observabilityclient
+      when: patch_observabilityclient | bool
+      tags:
+        - setup
+
+    - name: "Run Telemetry Verify Metrics tests"
+      ansible.builtin.import_role:
+        name: telemetry_verify_metrics

ci/vars-metric-verification-test.yml

@@ -0,0 +1,8 @@
+---
+post_deploy_00_fvt_verify_metrics:
+  source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_verify_metrics_osp18.yml"
+  type: playbook
+  config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg"
+post_deploy_99_collect_results:
+  source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml"
+  type: playbook

ci/vars/osp18_env.yml

@@ -5,4 +5,4 @@ stack_image: "cirros"
 stack_flavor: "m1.small"
 stack_network: "private"
 stack_external_network: "public"
-stack_name: "vnf"
+stack_name: "vnf"

roles/common/README.md

@@ -35,15 +35,14 @@ For pod_tests.yml tasks:
     common_pod_nspace
       - list of projects where pods exist
 
+
 For crd_tests.yml tasks:
 
     common_crd_test_id
       - polarion ID number for each test.
     common_crd_list
       - list of crd to validate
 
-
-
 For endpoint_tests.yml tasks:
 
     common_endpoint_test_id
@@ -66,6 +65,21 @@ For manifest_tests.yml tasks:
      manifest_list
        - list of package manifests to validate
 
+For cr\_tests.yml tasks:
+
+    common\_cr\_test\_id is defined
+       - polarion ID number for each test that a CR exists
+    common\_cr\_list is defined
+       - list of CRs to check
+        Each dict should include the following keys: kind, name
+	A dict can optionally include a "condition\_type" key.
+        Example:
+           kind: metricstorage
+           name: metric-storage
+           condition\_type: Ready
+    common\_cr\_ready\_test\_id is optionally defined
+       - polarion ID number for each test of readiness of the CR
+
 
 
 Dependencies

roles/common/tasks/container_test.yml

@@ -2,14 +2,13 @@
 - name: Get container status
   ansible.builtin.shell:
     cmd: |
-      podman ps -a --format "{{ '{{.Names}} {{.Status}}' }}" | grep {{ item }}
+      podman ps -a --format "{{ '{{.Names}} {{.Status}}' }}" | grep {{ container_name }}
   changed_when: false
   register: container_status
 
-
 - name: Verify container status {{ common_container_test_id }}
   ansible.builtin.assert:
     that:
-      - "'Up' in container_status.stdout"
-    success_msg: "Container '{{ item }}' is in 'Up' status."
-    fail_msg: "Container '{{ item }}' is not in 'Up' status. Current status: {{ container_status.stdout }}"
+      - "'unhealthy' not in container_status.stdout"
+    success_msg: "Container '{{ container_name }}' is in 'healthy' status."
+    fail_msg: "Container '{{ container_name }}' is not in 'healthy' status. Current status: {{ container_status.stdout }}"

roles/common/tasks/cr_tests.yml

@@ -0,0 +1,20 @@
+- name: Verify that a CR exists {{ common_cr_test_id }}
+  ansible.builtin.command:
+    cmd: |
+      oc get {{ item.kind }} {{ item.name }}
+  register: result
+  changed_when: false
+  failed_when:
+    - result.rc != 0
+
+- name: Verify that a CR is ready {{ common_cr_ready_test_id }}
+  ansible.builtin.command:
+    cmd: |
+      oc get {{ item.kind }} {{ item.name }} -o jsonpath='{.status.conditions[?(@.type=="{{ item.condition_type }}")].status}{"\n"}'
+  register: result
+  changed_when: false
+  failed_when:
+    - result.stdout != "True"
+  when:
+    - common_cr_ready_test_id is defined
+    - item.condition_type is defined

roles/common/tasks/main.yml

@@ -35,10 +35,18 @@
   ansible.builtin.include_tasks: "crd_tests.yml"
   loop: "{{ common_crd_list }}"
 
+- name: "Run CR tests"
+  when:
+    - common_cr_test_id is defined
+    - common_cr_list is defined
+  ansible.builtin.include_tasks: "cr_tests.yml"
+  loop: "{{ common_cr_list }}"
+
 - name: "Verify container tests"
   when:
     - common_container_list is defined
     - common_container_test_id is defined
   ansible.builtin.include_tasks: "container_test.yml"
   loop: "{{ common_container_list }}"
-
+  loop_control:
+    loop_var: container_name

roles/telemetry_verify_metrics/README.md

@@ -0,0 +1,51 @@
+telemetry_verify_metrics
+=========
+
+Test that expected metrics appear in Prometheus
+
+Requirements
+------------
+OpenStack deployed with the following enabled:
+- telemetry
+- metricstorage
+- ceilometer
+- rabbitmq
+
+Tests:
+------
+- Verify OpenStack is deployed correctly
+    - Verify telemetry is ready
+    - Verify metricstorage is ready
+    - Verify ceilometer is ready
+    - Verify rabbitmq is ready
+- Verify RabbitMQ metrics are being exposed and stored
+    - Check the rabbitmq metrics endpoint
+    - Use openstack observabilityclient to verify RabbitMQ metrics are stored in Prometheus
+- Verify Ceilometer metrics are being exposed and stored
+    - Use openstack observabilityclient to verify Ceilometer central metrics are stored in Prometheus
+    - Use openstack observabilityclient to verify Ceilometer compute metrics are stored in Prometheus
+- Verify NodeExporter metrics are being exposed and stored
+    - Use openstack observabilityclient to verify NodeExporter metrics are stored in Prometheus
+
+Role Variables
+--------------
+openstack\_cmd - command to access openstack cli. For example: "oc rsh openstackclient openstack"
+telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, node\_exporter, rabbitmq
+
+Example Playbook
+----------------
+- name: Run telemetry tests to verify metrics on osp18
+  hosts:  "{{ cifmw\_target\_hook\_host | default('localhost')  }}"
+  gather\_facts: true
+  environment:
+    KUBECONFIG: "path to kubeconfig"
+    PATH: "PATH variable contents"
+  tasks
+    - name: "Run Telemetry Verify Metrics tests"
+      ansible.builtin.import_role:
+        name: telemetry_verify_metrics
+
+License
+-------
+
+Apache 2

roles/telemetry_verify_metrics/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+telemetry_verify_metrics_metric_sources_to_test:
+  - ceilometer_compute_agent
+  - ceilometer_central_agent
+# Disable node exporter testing until OSPRH-11059 is fixed
+#  - node_exporter
+  - rabbitmq

roles/telemetry_verify_metrics/meta/main.yml

@@ -0,0 +1,52 @@
+galaxy_info:
+  author: Jaromir Wysoglad
+  description: Test that metrics from all sources are stored in Prometheus
+  company: Red Hat
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: Apache-2.0
+
+  min_ansible_version: "2.1"
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

roles/telemetry_verify_metrics/tasks/check_compute_node_containers.yml

@@ -0,0 +1,7 @@
+- delegate_to: "{{ compute_node }}"
+  # The containers on compute nodes seem to run on the root user, so we need to connect as root
+  become: true
+  block:
+  - name: Check compute node containers are up for {{ compute_node }}
+    ansible.builtin.include_role:
+      name: common

roles/telemetry_verify_metrics/tasks/main.yml

@@ -0,0 +1,48 @@
+---
+- name: Verify OpenStack is deployed correctly
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_test_id: RHOSO-1258
+    common_cr_ready_test_id: RHOSO-1259
+    common_cr_list:
+      - kind: telemetry
+        name: telemetry
+        condition_type: Ready
+      - kind: metricstorage
+        name: metric-storage
+        condition_type: Ready
+      - kind: ceilometer
+        name: ceilometer
+        condition_type: Ready
+      - kind: rabbitmq
+        name: rabbitmq
+        condition_type: ReconcileSuccess
+      - kind: rabbitmq
+        name: rabbitmq-cell1
+        condition_type: ReconcileSuccess
+  tags: precheck
+
+- name: Verify RabbitMQ metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_rabbitmq_metrics.yml
+  tags: test
+  when: '"rabbitmq" in telemetry_verify_metrics_metric_sources_to_test'
+
+- name: Verify Ceilometer compute metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_ceilometer_compute_metrics.yml
+  tags: test
+  when: '"ceilometer_compute_agent" in telemetry_verify_metrics_metric_sources_to_test'
+
+- name: Verify Ceilometer central metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_ceilometer_central_metrics.yml
+  tags: test
+  when: '"ceilometer_central_agent" in telemetry_verify_metrics_metric_sources_to_test'
+
+- name: Verify NodeExporter metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_node_exporter_metrics.yml
+  tags: test
+  when: '"node_exporter" in telemetry_verify_metrics_metric_sources_to_test'

roles/telemetry_verify_metrics/tasks/verify_ceilometer_central_metrics.yml

@@ -0,0 +1,46 @@
+- name: Verify ceilometer scrapeconfig exists
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_test_id: RHOSO-1220
+    common_cr_list:
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-ceilometer
+
+- name: Verify ceilometer central agent is running
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_pod_test_id: RHOSO-1240
+    common_pod_status_str: "Running"
+    common_pod_nspace: openstack
+    common_pod_list:
+      - ceilometer-0
+
+- block:
+    - name: Create an image
+      ansible.builtin.shell: |
+        curl -L -# http://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-disk.img > /tmp/fvt_testing_image.img
+        {{ openstack_cmd }} image create --container-format bare --disk-format qcow2 fvt_central_testing_image < /tmp/fvt_testing_image.img
+      register: result
+      changed_when: result.rc == 0
+      failed_when: result.rc >= 1
+
+    - name: |
+        TEST Use openstack observabilityclient to verify ceilometer central metrics are stored in prometheus
+        RHOSO-1212
+      ansible.builtin.shell: |
+        {{ openstack_cmd }} metric show ceilometer_image_size
+      register: result
+      delay: 30
+      retries: 10
+      until: result.rc == 0 and "ceilometer_image_size" in result.stdout
+      changed_when: false
+
+  always:
+    - name: Delete the image
+      ansible.builtin.shell: |
+        {{ openstack_cmd }} image show fvt_central_testing_image && {{ openstack_cmd }} image delete fvt_central_testing_image
+      register: result
+      changed_when: result.rc == 0
+      failed_when: false