Skip to content

Commit

Permalink
Merge pull request #519 from vitessio/prometheus-hot-reload
Browse files Browse the repository at this point in the history 
Fix lack of Prometheus metrics and improve reliability of collected metrics
  • Loading branch information
@frouioui
frouioui authored Apr 2, 2024
2 parents c2685d3 + cd00b84 commit fcde460
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 131 deletions.
8 changes: 0 additions & 8 deletions ansible/clean_macrobench.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,5 @@
# limitations under the License.

---
- name: Stop prometheus
hosts:
- all
roles:
- role: prometheus
vars:
- prom_start: False

- name: Teardown Cluster
import_playbook: teardown_cluster.yml
107 changes: 7 additions & 100 deletions ansible/macrobench.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@
# limitations under the License.

---
- hosts: prometheus
tasks:
- name: Update Prometheus labels
include_role:
name: prometheus
tasks_from: update_uuid

- hosts: all
tasks:
- name: Ensure users are created
Expand All @@ -32,106 +39,6 @@
name: sysbench
tasks_from: install

- hosts: prometheus
become: yes
become_user: root
tags:
- prometheus
roles:
- { role: cloudalchemy.prometheus, when: stats_remote_db_host is defined and arewefastyet_last_exec_is_same is undefined }
vars:
vitess_cluster: 'main'
mysqld_targets: "{% set targets = [] -%}{% for host in groups['vttablet'] -%}{% for tablet in hostvars[host]['tablets'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (tablet.mysqld_exporter_port | default(9104) |string )) }}{% endfor -%}{% endfor -%}{{ targets }}"
tablet_targets: "{% set targets = [] -%}{% for host in groups['vttablet'] -%}{% for tablet in hostvars[host]['tablets'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (tablet.port | default(16001) | string )) }}{% endfor -%}{% endfor -%}{{ targets }}"
gateway_targets: "{% set targets = [] -%}{% for host in groups['vtgate'] -%}{% for gateway in hostvars[host]['gateways'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (gateway.port | default(15001) | string )) }}{% endfor -%}{% endfor -%}{{ targets }}"
vtctld_targets: "{% set targets = [] -%}{% for host in groups['vtctld'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (vtctld_port | default(15000) | string )) }}{% endfor -%}{{ targets }}"
node_targets: "{% set targets= [] -%}{% for host in groups['all'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':9100' ) }}{% endfor -%}{{ targets }}"
prometheus_skip_install: false
prometheus_version: 2.49.1
prometheus_global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
prometheus_targets:
tablets:
- targets: '{{ tablet_targets }}'
labels:
component: vttablet
app: vitess
cluster: '{{ vitess_cluster }}'
region: '{{ cell }}'
keyspace: '{{ keyspace }}'
exec_uuid: "{{ arewefastyet_exec_uuid }}"
mysqld:
- targets: '{{ mysqld_targets }}'
labels:
component: mysqld
app: vitess
cluster: '{{ vitess_cluster }}'
region: '{{ cell }}'
keyspace: '{{ keyspace }}'
exec_uuid: "{{ arewefastyet_exec_uuid }}"
gateways:
- targets: '{{ gateway_targets }}'
labels:
component: vtgate
app: vitess
cluster: '{{ vitess_cluster }}'
region: '{{ cell }}'
exec_uuid: "{{ arewefastyet_exec_uuid }}"
vtctld:
- targets: '{{ vtctld_targets }}'
labels:
component: vtctld
app: vitess
cluster: '{{ vitess_cluster }}'
region: '{{ cell }}'
exec_uuid: "{{ arewefastyet_exec_uuid }}"
nodes:
- targets: '{{ node_targets }}'
labels:
component: node
app: vitess
cluster: '{{ vitess_cluster }}'
exec_uuid: "{{ arewefastyet_exec_uuid }}"
prometheus_scrape_configs:
- job_name: "node"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/nodes.yml"
- job_name: "vitess-vttablet"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/tablets.yml"
- job_name: "mysql"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/mysqld.yml"
- job_name: "vitess-vtctld"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/vtctld.yml"
- job_name: "vitess-vtgate"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/gateways.yml"
prometheus_remote_write:
- url: "http://{{ stats_remote_db_host }}/prom/api/v1/write"
basic_auth:
username: "{{ stats_remote_db_user }}"
password: "{{ stats_remote_db_password }}"
- url: "http://{{ stats_remote_db_host }}:{{ stats_remote_db_port }}/api/v1/prom/write?db={{ stats_remote_db_database }}&u={{ stats_remote_db_user }}&p={{ stats_remote_db_password }}"
prometheus_external_labels:
exec_uuid: "{{ arewefastyet_exec_uuid }}"

- hosts: prometheus
tasks:
- name: Update Prometheus labels
when: arewefastyet_last_exec_is_same is defined
include_role:
name: prometheus
tasks_from: update_uuid

- hosts: macrobench
roles:
- macrobench
Expand Down
19 changes: 7 additions & 12 deletions ansible/roles/prometheus/tasks/update_uuid.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,13 @@
---
- name: Replace prometheus exec_uuid label value
with_items:
- 'gateways.yml'
- 'mysqld.yml'
- 'nodes.yml'
- 'tablets.yml'
- 'vtctld.yml'
- 'prometheus.yml'
- 'file_sd/gateways.yml'
- 'file_sd/mysqld.yml'
- 'file_sd/nodes.yml'
- 'file_sd/tablets.yml'
- 'file_sd/vtctld.yml'
lineinfile:
path: '/etc/prometheus/file_sd/{{ item }}'
path: '/etc/prometheus/{{ item }}'
regexp: '^(.*)exec_uuid:(.*)$'
line: ' exec_uuid: {{arewefastyet_exec_uuid}}'

- name: Restart prometheus service
service:
name: prometheus
state: restarted
enabled: yes
87 changes: 76 additions & 11 deletions ansible/setup.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,24 +57,89 @@
tags:
- prometheus
roles:
- cloudalchemy.prometheus
- { role: cloudalchemy.prometheus, when: stats_remote_db_host is defined }
vars:
vitess_cluster: 'main'
mysqld_targets: "{% set targets = [] -%}{% for host in groups['vttablet'] -%}{% for tablet in hostvars[host]['tablets'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (tablet.mysqld_exporter_port | default(9104) |string )) }}{% endfor -%}{% endfor -%}{{ targets }}"
tablet_targets: "{% set targets = [] -%}{% for host in groups['vttablet'] -%}{% for tablet in hostvars[host]['tablets'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (tablet.port | default(16001) | string )) }}{% endfor -%}{% endfor -%}{{ targets }}"
gateway_targets: "{% set targets = [] -%}{% for host in groups['vtgate'] -%}{% for gateway in hostvars[host]['gateways'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (gateway.port | default(15001) | string )) }}{% endfor -%}{% endfor -%}{{ targets }}"
vtctld_targets: "{% set targets = [] -%}{% for host in groups['vtctld'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (vtctld_port | default(15000) | string )) }}{% endfor -%}{{ targets }}"
node_targets: "{% set targets= [] -%}{% for host in groups['all'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':9100' ) }}{% endfor -%}{{ targets }}"
prometheus_skip_install: false
prometheus_version: 2.49.1
prometheus_global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
prometheus_targets:
tablets:
- targets: '{{ tablet_targets }}'
labels:
component: vttablet
app: vitess
cluster: '{{ vitess_cluster }}'
region: '{{ cell }}'
keyspace: '{{ keyspace }}'
exec_uuid: "exec_uuid"
mysqld:
- targets: '{{ mysqld_targets }}'
labels:
component: mysqld
app: vitess
cluster: '{{ vitess_cluster }}'
region: '{{ cell }}'
keyspace: '{{ keyspace }}'
exec_uuid: "exec_uuid"
gateways:
- targets: '{{ gateway_targets }}'
labels:
component: vtgate
app: vitess
cluster: '{{ vitess_cluster }}'
region: '{{ cell }}'
exec_uuid: "exec_uuid"
vtctld:
- targets: '{{ vtctld_targets }}'
labels:
component: vtctld
app: vitess
cluster: '{{ vitess_cluster }}'
region: '{{ cell }}'
exec_uuid: "exec_uuid"
nodes:
- targets: '{{ node_targets }}'
labels:
component: node
- targets: '{{ node_targets }}'
labels:
component: node
app: vitess
cluster: '{{ vitess_cluster }}'
exec_uuid: "exec_uuid"
prometheus_scrape_configs:
- job_name: "node"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/nodes.yml"
- job_name: "vitess-vttablet"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/tablets.yml"
- job_name: "mysql"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/mysqld.yml"
- job_name: "vitess-vtctld"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/vtctld.yml"
- job_name: "vitess-vtgate"
file_sd_configs:
- files:
- "{{ prometheus_config_dir }}/file_sd/gateways.yml"
prometheus_remote_write:
- url: "http://{{ stats_remote_db_host }}/prom/api/v1/write"
basic_auth:
username: "{{ stats_remote_db_user }}"
password: "{{ stats_remote_db_password }}"
- url: "http://{{ stats_remote_db_host }}:{{ stats_remote_db_port }}/api/v1/prom/write?db={{ stats_remote_db_database }}&u={{ stats_remote_db_user }}&p={{ stats_remote_db_password }}"
prometheus_external_labels:
exec_uuid: "exec_uuid"

- name: Stop prometheus
hosts:
- all
roles:
- role: prometheus
vars:
- prom_start: False

0 comments on commit fcde460

Please sign in to comment.