From ed4cd1419cab7c168b2e10fae4249b34069c0f28 Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Wed, 19 Aug 2020 15:47:29 -0400 Subject: [PATCH 1/8] Try directly querying prometheus from cost operator * Query prom api using cert and service token --- roles/collect/defaults/main.yml | 1 + roles/collect/tasks/main.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/roles/collect/defaults/main.yml b/roles/collect/defaults/main.yml index bea4c35..2ecac5a 100644 --- a/roles/collect/defaults/main.yml +++ b/roles/collect/defaults/main.yml @@ -25,3 +25,4 @@ debug: 'true' collect_upload_wait: '{{ 2100 | random(step=10) }}' current_month: '{{ ansible_date_time.month | int }}' current_year: '{{ ansible_date_time.year | int }}' +prom_url: 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query?query=up' \ No newline at end of file diff --git a/roles/collect/tasks/main.yml b/roles/collect/tasks/main.yml index 08ca3d5..62fa927 100644 --- a/roles/collect/tasks/main.yml +++ b/roles/collect/tasks/main.yml @@ -266,6 +266,20 @@ state: absent when: collect_delete_after | bool +- name: Get service account token + set_fact: + service_account_token: "{{ lookup('file', '/var/run/secrets/kubernetes.io/serviceaccount/token') }}" + +- name: Query Prometheus directly + shell: + cmd: 'curl -vvvv {{ prom_url }} -H "Authorization: Bearer {{ service_account_token }}" --cacert {{ cacert_path }}' + register: prom_output + +- name: debug prom output + debug: + var: prom_output + when: debug + - name: Create temp dir for downloaded files file: path: '{{ collect_cluster_download_path }}' From 5ce0da31ea199c37eb1a6c4cf6b73edb17d0a231 Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Wed, 19 Aug 2020 15:55:19 -0400 Subject: [PATCH 2/8] fix lint error --- roles/collect/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/collect/defaults/main.yml b/roles/collect/defaults/main.yml index 2ecac5a..36e8699 100644 --- a/roles/collect/defaults/main.yml +++ b/roles/collect/defaults/main.yml @@ -25,4 +25,4 @@ debug: 'true' collect_upload_wait: '{{ 2100 | random(step=10) }}' current_month: '{{ ansible_date_time.month | int }}' current_year: '{{ ansible_date_time.year | int }}' -prom_url: 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query?query=up' \ No newline at end of file +prom_url: 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query?query=up' From fcec48ae45f2d7b566f031567f9e51a3c755aba2 Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Wed, 19 Aug 2020 18:27:28 -0400 Subject: [PATCH 3/8] remove ansible container log collection on rescue. use -k option with curl --- molecule/test-local/converge.yml | 42 -------------------------------- roles/collect/tasks/main.yml | 2 +- 2 files changed, 1 insertion(+), 43 deletions(-) diff --git a/molecule/test-local/converge.yml b/molecule/test-local/converge.yml index 9a1d700..f4623c2 100644 --- a/molecule/test-local/converge.yml +++ b/molecule/test-local/converge.yml @@ -207,20 +207,6 @@ - debug: var=log.stdout_lines - - name: get ansible logs - ignore_errors: yes - failed_when: false - command: kubectl logs - deployment/{{ definition.metadata.name }} -n {{ namespace }} -c ansible - environment: - KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}' - vars: - definition: "{{ lookup('template', - '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}" - register: log - - - debug: var=log.stdout_lines - - fail: msg: "Failed on action: converge" @@ -402,20 +388,6 @@ - debug: var=log.stdout_lines - - name: get ansible logs - ignore_errors: yes - failed_when: false - command: kubectl logs - deployment/{{ definition.metadata.name }} -n {{ namespace }} -c ansible - environment: - KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}' - vars: - definition: "{{ lookup('template', - '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}" - register: log - - - debug: var=log.stdout_lines - - fail: msg: "Failed on action: converge" @@ -571,20 +543,6 @@ - debug: var=log.stdout_lines - - name: get ansible logs - ignore_errors: yes - failed_when: false - command: kubectl logs - deployment/{{ definition.metadata.name }} -n {{ namespace }} -c ansible - environment: - KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}' - vars: - definition: "{{ lookup('template', - '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}" - register: log - - - debug: var=log.stdout_lines - - fail: msg: "Failed on action: converge" diff --git a/roles/collect/tasks/main.yml b/roles/collect/tasks/main.yml index 62fa927..9bb3420 100644 --- a/roles/collect/tasks/main.yml +++ b/roles/collect/tasks/main.yml @@ -272,7 +272,7 @@ - name: Query Prometheus directly shell: - cmd: 'curl -vvvv {{ prom_url }} -H "Authorization: Bearer {{ service_account_token }}" --cacert {{ cacert_path }}' + cmd: 'curl -vvvv -k {{ prom_url }} -H "Authorization: Bearer {{ service_account_token }}" --cacert {{ cacert_path }}' register: prom_output - name: debug prom output From c7fa9fb6fb03660ac2e2fe07ed4c659a5a4345e8 Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Thu, 20 Aug 2020 15:10:22 -0400 Subject: [PATCH 4/8] Update operator with python script query --- Makefile | 8 ++- deploy/cluster_role.yaml | 53 ++++++++++++++++++ deploy/cluster_role_binding.yaml | 13 +++++ roles/collect/files/query_prom.py | 89 +++++++++++++++++++++++++++++++ roles/collect/tasks/main.yml | 4 ++ 5 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 deploy/cluster_role.yaml create mode 100644 deploy/cluster_role_binding.yaml create mode 100644 roles/collect/files/query_prom.py diff --git a/Makefile b/Makefile index fd2b9ec..d7cdb81 100644 --- a/Makefile +++ b/Makefile @@ -58,13 +58,19 @@ deploy-operator-dev-branch: sed -i "" "s?{{ pull_policy|default('Always') }}?Always?g" testing/operator.yaml oc apply -f testing/operator.yaml -deploy-dependencies: +setup-cluster-role: + @cp deploy cluster_role.yaml testing/cluster_role.yaml + @sed -i "" 's/placeholder/$(shell echo $(or $(namespace),openshift-metering))/g' testing/cluster_role.yaml + +deploy-dependencies: setup-cluster-role oc apply -f deploy/crds/cost_mgmt_crd.yaml || true oc apply -f deploy/crds/cost_mgmt_data_crd.yaml || true oc apply -f testing/authentication_secret.yaml oc apply -f deploy/service_account.yaml oc apply -f deploy/role.yaml oc apply -f deploy/role_binding.yaml + oc apply -f testing/cluster_role.yaml + oc apply -f deploy/cluster_role_binding.yaml deploy-custom-resources: oc apply -f testing/cost_mgmt_cr.yaml diff --git a/deploy/cluster_role.yaml b/deploy/cluster_role.yaml new file mode 100644 index 0000000..6b10fcd --- /dev/null +++ b/deploy/cluster_role.yaml @@ -0,0 +1,53 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openshift-cost-mgmt-operator +rules: + - verbs: + - update + apiGroups: + - '' + resources: + - services/finalizers + - deployments/finalizers + - verbs: + - get + - list + apiGroups: + - '' + resources: + - namespaces + - verbs: + - list + - get + apiGroups: + - config.openshift.io + resources: + - proxies + - verbs: + - list + - get + apiGroups: + - config.openshift.io + resources: + - networks + - verbs: + - create + apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + - verbs: + - create + apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + - verbs: + - '*' + apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles diff --git a/deploy/cluster_role_binding.yaml b/deploy/cluster_role_binding.yaml new file mode 100644 index 0000000..e241e24 --- /dev/null +++ b/deploy/cluster_role_binding.yaml @@ -0,0 +1,13 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openshift-cost-operator-rb +subjects: + - kind: ServiceAccount + name: cost-mgmt-operator + namespace: placeholder +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: openshift-cost-mgmt-operator \ No newline at end of file diff --git a/roles/collect/files/query_prom.py b/roles/collect/files/query_prom.py new file mode 100644 index 0000000..1b8f7b6 --- /dev/null +++ b/roles/collect/files/query_prom.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Red Hat, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +"""Query Prometheus for OpenShift cluster metrics.""" + +import argparse +import csv +import datetime +import logging +import json +import requests +import sys +import time + + +# default prometheus query +DEFAULT_PROMETHEUS = 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query_range' +DEFAULT_QUERY = "up" + +# logging +LOG = logging.getLogger(__name__) +LOG_FORMAT = "%(asctime)s [%(levelname)s] %(message)s" +LOG_VERBOSITY = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] +logging.basicConfig(format=LOG_FORMAT, level=logging.ERROR, stream=sys.stdout) + + +def parse_args(): + """Handle CLI arg parsing.""" + parser = argparse.ArgumentParser( + description="Cost Management prometheus query script", prog=sys.argv[0]) + + # required args + parser.add_argument("-c", "--cacert", required=True, + help="path to cacert file") + + parser.add_argument("-b", "--bearer", required=True, + help="Bearer token value") + + parser.add_argument("-p", "--prometheus-url", required=True, + default=DEFAULT_PROMETHEUS, help="Prometheus host and query api") + + parser.add_argument("-q", "--query", required=True, + default=DEFAULT_QUERY, help="Prometheus query") + + parser.add_argument("-v", "--verbosity", action="count", + default=0, help="increase verbosity (up to -vvv)") + return parser.parse_args() + + +def execute_prom_query(prometheus_url, query): + """Query prometheus for metrics.""" + results = None + end = datetime.datetime.now().replace(minute=0, second=0, microsecond=0) + start = end - datetime.timedelta(hours=1) + step = '1h' + req_params={ + 'query': query, + 'start': start, + 'end': end, + 'step': step} + response = requests.get(prometheus_url, params=req_params) + results = response.json()['data']['result'] + + return results + + +if "__main__" in __name__: + args = parse_args() + if args.verbosity: + LOG.setLevel(LOG_VERBOSITY[args.verbosity]) + LOG.debug("CLI Args: %s", args) + + json_results = execute_prom_query(args.prometheus_url, args.query) + print(json_results) \ No newline at end of file diff --git a/roles/collect/tasks/main.yml b/roles/collect/tasks/main.yml index 9bb3420..f168ed0 100644 --- a/roles/collect/tasks/main.yml +++ b/roles/collect/tasks/main.yml @@ -280,6 +280,10 @@ var: prom_output when: debug +- name: Run Prometheus query script + script: query_prom.py --cacert {{ cacert_path }} --bearer {{ service_account_token }} + register: query_prom + - name: Create temp dir for downloaded files file: path: '{{ collect_cluster_download_path }}' From bac5b4a2f1fef59899b69d72799af758ba5c04cb Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Thu, 20 Aug 2020 16:31:30 -0400 Subject: [PATCH 5/8] remove required=True from optional params --- roles/collect/files/query_prom.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/collect/files/query_prom.py b/roles/collect/files/query_prom.py index 1b8f7b6..2ab2a4c 100644 --- a/roles/collect/files/query_prom.py +++ b/roles/collect/files/query_prom.py @@ -51,10 +51,10 @@ def parse_args(): parser.add_argument("-b", "--bearer", required=True, help="Bearer token value") - parser.add_argument("-p", "--prometheus-url", required=True, + parser.add_argument("-p", "--prometheus-url", default=DEFAULT_PROMETHEUS, help="Prometheus host and query api") - parser.add_argument("-q", "--query", required=True, + parser.add_argument("-q", "--query", default=DEFAULT_QUERY, help="Prometheus query") parser.add_argument("-v", "--verbosity", action="count", From 7f4407bb372962adbbe211e3fa9ae84725167de3 Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Fri, 21 Aug 2020 09:07:29 -0400 Subject: [PATCH 6/8] display results --- roles/collect/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/collect/tasks/main.yml b/roles/collect/tasks/main.yml index f168ed0..6e09326 100644 --- a/roles/collect/tasks/main.yml +++ b/roles/collect/tasks/main.yml @@ -284,6 +284,11 @@ script: query_prom.py --cacert {{ cacert_path }} --bearer {{ service_account_token }} register: query_prom +- name: debug query prom + debug: + var: query_prom + when: debug + - name: Create temp dir for downloaded files file: path: '{{ collect_cluster_download_path }}' From 2fd8cce944baa0979141af03315ff05a6a75346d Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Fri, 21 Aug 2020 13:06:50 -0400 Subject: [PATCH 7/8] pass cert and token on prom query call. --- roles/collect/files/query_prom.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/collect/files/query_prom.py b/roles/collect/files/query_prom.py index 2ab2a4c..19569dc 100644 --- a/roles/collect/files/query_prom.py +++ b/roles/collect/files/query_prom.py @@ -62,7 +62,7 @@ def parse_args(): return parser.parse_args() -def execute_prom_query(prometheus_url, query): +def execute_prom_query(prometheus_url, query, cacert, bearer_token): """Query prometheus for metrics.""" results = None end = datetime.datetime.now().replace(minute=0, second=0, microsecond=0) @@ -73,7 +73,8 @@ def execute_prom_query(prometheus_url, query): 'start': start, 'end': end, 'step': step} - response = requests.get(prometheus_url, params=req_params) + headers = {"Authorization": f"Bearer {bearer_token}"} + response = requests.get(prometheus_url, params=req_params, cert=cacert, verify=False, headers=headers) results = response.json()['data']['result'] return results @@ -85,5 +86,5 @@ def execute_prom_query(prometheus_url, query): LOG.setLevel(LOG_VERBOSITY[args.verbosity]) LOG.debug("CLI Args: %s", args) - json_results = execute_prom_query(args.prometheus_url, args.query) + json_results = execute_prom_query(args.prometheus_url, args.query, args.cacert, args.bearer) print(json_results) \ No newline at end of file From afd2f9efaf4ff193a5b314be270072f056eca75b Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Fri, 21 Aug 2020 16:05:19 -0400 Subject: [PATCH 8/8] Fix timestamps for query. --- roles/collect/files/query_prom.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/roles/collect/files/query_prom.py b/roles/collect/files/query_prom.py index 19569dc..8a909ed 100644 --- a/roles/collect/files/query_prom.py +++ b/roles/collect/files/query_prom.py @@ -29,7 +29,7 @@ # default prometheus query -DEFAULT_PROMETHEUS = 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query_range' +DEFAULT_PROMETHEUS = "https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query_range" DEFAULT_QUERY = "up" # logging @@ -65,17 +65,19 @@ def parse_args(): def execute_prom_query(prometheus_url, query, cacert, bearer_token): """Query prometheus for metrics.""" results = None - end = datetime.datetime.now().replace(minute=0, second=0, microsecond=0) - start = end - datetime.timedelta(hours=1) - step = '1h' + end_dt = datetime.datetime.utcnow().replace(minute=0, second=0, microsecond=0) + start_dt = end - datetime.timedelta(hours=1) + end = end_dt.isoformat("T") + "Z" + start = start_dt.isoformat("T") + "Z" + step = "1h" req_params={ - 'query': query, - 'start': start, - 'end': end, - 'step': step} + "query": query, + "start": start, + "end": end, + "step": step} headers = {"Authorization": f"Bearer {bearer_token}"} - response = requests.get(prometheus_url, params=req_params, cert=cacert, verify=False, headers=headers) - results = response.json()['data']['result'] + response = requests.get(prometheus_url, params=req_params, verify=False, headers=headers) + results = response.json()["data"]["result"] return results