WIP: Try directly querying prometheus from cost operator

Makefile

@@ -58,13 +58,19 @@ deploy-operator-dev-branch:
 	sed -i "" "s?{{ pull_policy|default('Always') }}?Always?g" testing/operator.yaml
 	oc apply -f testing/operator.yaml
 
-deploy-dependencies:
+setup-cluster-role:
+	@cp deploy cluster_role.yaml testing/cluster_role.yaml
+	@sed -i "" 's/placeholder/$(shell echo $(or $(namespace),openshift-metering))/g' testing/cluster_role.yaml
+
+deploy-dependencies: setup-cluster-role
 	oc apply -f deploy/crds/cost_mgmt_crd.yaml || true
 	oc apply -f deploy/crds/cost_mgmt_data_crd.yaml || true
 	oc apply -f testing/authentication_secret.yaml
 	oc apply -f deploy/service_account.yaml
 	oc apply -f deploy/role.yaml
 	oc apply -f deploy/role_binding.yaml
+	oc apply -f testing/cluster_role.yaml
+	oc apply -f deploy/cluster_role_binding.yaml
 
 deploy-custom-resources:
 	oc apply -f testing/cost_mgmt_cr.yaml

deploy/cluster_role.yaml

@@ -0,0 +1,53 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: openshift-cost-mgmt-operator
+rules:
+  - verbs:
+      - update
+    apiGroups:
+      - ''
+    resources:
+      - services/finalizers
+      - deployments/finalizers
+  - verbs:
+      - get
+      - list
+    apiGroups:
+      - ''
+    resources:
+      - namespaces
+  - verbs:
+      - list
+      - get
+    apiGroups:
+      - config.openshift.io
+    resources:
+      - proxies
+  - verbs:
+      - list
+      - get
+    apiGroups:
+      - config.openshift.io
+    resources:
+      - networks
+  - verbs:
+      - create
+    apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+  - verbs:
+      - create
+    apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+  - verbs:
+      - '*'
+    apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - clusterrolebindings
+      - clusterroles

deploy/cluster_role_binding.yaml

@@ -0,0 +1,13 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: openshift-cost-operator-rb
+subjects:
+  - kind: ServiceAccount
+    name: cost-mgmt-operator
+    namespace: placeholder
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: openshift-cost-mgmt-operator

molecule/test-local/converge.yml

@@ -207,20 +207,6 @@
 
         - debug: var=log.stdout_lines
 
-        - name: get ansible logs
-          ignore_errors: yes
-          failed_when: false
-          command: kubectl logs
-            deployment/{{ definition.metadata.name }} -n {{ namespace }} -c ansible
-          environment:
-            KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}'
-          vars:
-            definition: "{{ lookup('template',
-              '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}"
-          register: log
-
-        - debug: var=log.stdout_lines
-
         - fail:
             msg: "Failed on action: converge"
 
@@ -402,20 +388,6 @@
 
         - debug: var=log.stdout_lines
 
-        - name: get ansible logs
-          ignore_errors: yes
-          failed_when: false
-          command: kubectl logs
-            deployment/{{ definition.metadata.name }} -n {{ namespace }} -c ansible
-          environment:
-            KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}'
-          vars:
-            definition: "{{ lookup('template',
-              '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}"
-          register: log
-
-        - debug: var=log.stdout_lines
-
         - fail:
             msg: "Failed on action: converge"
 
@@ -571,20 +543,6 @@
 
         - debug: var=log.stdout_lines
 
-        - name: get ansible logs
-          ignore_errors: yes
-          failed_when: false
-          command: kubectl logs
-            deployment/{{ definition.metadata.name }} -n {{ namespace }} -c  ansible
-          environment:
-            KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}'
-          vars:
-            definition: "{{ lookup('template',
-              '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}"
-          register: log
-
-        - debug: var=log.stdout_lines
-
         - fail:
             msg: "Failed on action: converge"
 

roles/collect/defaults/main.yml

@@ -25,3 +25,4 @@ debug: 'true'
 collect_upload_wait: '{{ 2100 | random(step=10) }}'
 current_month: '{{ ansible_date_time.month | int }}'
 current_year: '{{ ansible_date_time.year | int }}'
+prom_url: 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query?query=up'

roles/collect/files/query_prom.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 Red Hat, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+"""Query Prometheus for OpenShift cluster metrics."""
+
+import argparse
+import csv
+import datetime
+import logging
+import json
+import requests
+import sys
+import time
+
+
+# default prometheus query
+DEFAULT_PROMETHEUS = "https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query_range"
+DEFAULT_QUERY = "up"
+
+# logging
+LOG = logging.getLogger(__name__)
+LOG_FORMAT = "%(asctime)s [%(levelname)s] %(message)s"
+LOG_VERBOSITY = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
+logging.basicConfig(format=LOG_FORMAT, level=logging.ERROR, stream=sys.stdout)
+
+
+def parse_args():
+    """Handle CLI arg parsing."""
+    parser = argparse.ArgumentParser(
+        description="Cost Management prometheus query script", prog=sys.argv[0])
+
+    # required args
+    parser.add_argument("-c", "--cacert", required=True,
+                        help="path to cacert file")
+
+    parser.add_argument("-b", "--bearer", required=True,
+                        help="Bearer token value")
+
+    parser.add_argument("-p", "--prometheus-url",
+                        default=DEFAULT_PROMETHEUS, help="Prometheus host and query api")
+
+    parser.add_argument("-q", "--query",
+                        default=DEFAULT_QUERY, help="Prometheus query")
+
+    parser.add_argument("-v", "--verbosity", action="count",
+                        default=0, help="increase verbosity (up to -vvv)")
+    return parser.parse_args()
+
+
+def execute_prom_query(prometheus_url, query, cacert, bearer_token):
+    """Query prometheus for metrics."""
+    results = None
+    end_dt = datetime.datetime.utcnow().replace(minute=0, second=0, microsecond=0)
+    start_dt = end - datetime.timedelta(hours=1)
+    end = end_dt.isoformat("T") + "Z"
+    start = start_dt.isoformat("T") + "Z"
+    step = "1h"
+    req_params={ 
+        "query": query,
+        "start": start,
+        "end": end,
+        "step": step}
+    headers = {"Authorization": f"Bearer {bearer_token}"}
+    response = requests.get(prometheus_url, params=req_params, verify=False, headers=headers)
+    results = response.json()["data"]["result"]
+
+    return results
+
+
+if "__main__" in __name__:
+    args = parse_args()
+    if args.verbosity:
+        LOG.setLevel(LOG_VERBOSITY[args.verbosity])
+    LOG.debug("CLI Args: %s", args)
+
+    json_results = execute_prom_query(args.prometheus_url, args.query, args.cacert, args.bearer)
+    print(json_results)

roles/collect/tasks/main.yml

@@ -266,6 +266,29 @@
     state: absent
   when: collect_delete_after | bool
 
+- name: Get service account token
+  set_fact:
+    service_account_token: "{{ lookup('file', '/var/run/secrets/kubernetes.io/serviceaccount/token') }}"
+
+- name: Query Prometheus directly
+  shell:
+    cmd: 'curl -vvvv -k {{ prom_url }} -H "Authorization: Bearer {{ service_account_token }}" --cacert {{ cacert_path }}'
+  register: prom_output
+
+- name: debug prom output
+  debug:
+    var: prom_output
+  when: debug
+
+- name: Run Prometheus query script
+  script: query_prom.py --cacert {{ cacert_path }} --bearer {{ service_account_token }}
+  register: query_prom
+
+- name: debug query prom
+  debug:
+    var: query_prom
+  when: debug
+
 - name: Create temp dir for downloaded files
   file:
     path: '{{ collect_cluster_download_path }}'