diff --git a/userspace/libsinsp/metrics_collector.cpp b/userspace/libsinsp/metrics_collector.cpp index e6e213ef60..f0b9aaf611 100644 --- a/userspace/libsinsp/metrics_collector.cpp +++ b/userspace/libsinsp/metrics_collector.cpp @@ -23,17 +23,18 @@ limitations under the License. #include #include #include +#include static const char *const sinsp_stats_v2_resource_utilization_names[] = { [SINSP_RESOURCE_UTILIZATION_CPU_PERC] = "cpu_usage_perc", - [SINSP_RESOURCE_UTILIZATION_MEMORY_RSS] = "memory_rss", - [SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ] = "memory_vsz", - [SINSP_RESOURCE_UTILIZATION_MEMORY_PSS] = "memory_pss", - [SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY] = "container_memory_used", - [SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST] = "cpu_usage_perc_total_host", - [SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST] = "memory_used_host", - [SINSP_RESOURCE_UTILIZATION_PROCS_HOST] = "procs_running_host", - [SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST] = "open_fds_host", + [SINSP_RESOURCE_UTILIZATION_MEMORY_RSS] = "memory_rss_kb", + [SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ] = "memory_vsz_kb", + [SINSP_RESOURCE_UTILIZATION_MEMORY_PSS] = "memory_pss_kb", + [SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY] = "container_memory_used_bytes", + [SINSP_RESOURCE_UTILIZATION_HOST_CPU_PERC] = "host_cpu_usage_perc", + [SINSP_RESOURCE_UTILIZATION_HOST_MEMORY] = "host_memory_used_kb", + [SINSP_RESOURCE_UTILIZATION_HOST_PROCS] = "host_procs_running", + [SINSP_RESOURCE_UTILIZATION_HOST_FDS] = "host_open_fds", [SINSP_STATS_V2_N_THREADS] = "n_threads", [SINSP_STATS_V2_N_FDS] = "n_fds", [SINSP_STATS_V2_NONCACHED_FD_LOOKUPS] = "n_noncached_fd_lookups", @@ -74,9 +75,174 @@ static const char *const metrics_metric_type_name_mappings_prometheus[] = { [METRIC_VALUE_NON_MONOTONIC_CURRENT] = "gauge", }; -namespace libsinsp::metrics { +namespace libs::metrics { -void metrics_collector::get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, uint32_t &vsz, uint32_t &pss, uint64_t &memory_used_host, uint64_t &open_fds_host) +std::string metrics_converter::convert_metric_to_text(metrics_v2 metric) +{ + std::string metric_text = std::string(metric.name) + " "; + switch (metric.type) + { + case METRIC_VALUE_TYPE_U32: + metric_text += std::to_string(metric.value.u32); + break; + case METRIC_VALUE_TYPE_S32: + metric_text += std::to_string(metric.value.s32); + break; + case METRIC_VALUE_TYPE_U64: + metric_text += std::to_string(metric.value.u64); + break; + case METRIC_VALUE_TYPE_S64: + metric_text += std::to_string(metric.value.s64); + break; + case METRIC_VALUE_TYPE_D: + metric_text += std::to_string(metric.value.d); + break; + case METRIC_VALUE_TYPE_F: + metric_text += std::to_string(metric.value.f); + break; + case METRIC_VALUE_TYPE_I: + metric_text += std::to_string(metric.value.i); + break; + default: + break; + } + metric_text += "\n"; + return metric_text; +} + +void metrics_converter::convert_metric_to_unit_convention(metrics_v2& metric) +{ + return; +} + +void output_rule_metrics_converter::convert_metric_to_unit_convention(metrics_v2& metric) +{ + RE2 unit_pattern("(_kb|_bytes)$"); + switch (metric.unit) + { + case METRIC_VALUE_UNIT_MEMORY_BYTES: + case METRIC_VALUE_UNIT_MEMORY_KIBIBYTES: + switch (metric.type) + { + case METRIC_VALUE_TYPE_U32: + { + metric.value.d = libs::metrics::convert_memory(metric.unit, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, metric.value.u32); + std::string metric_name_str(metric.name); + RE2::GlobalReplace(&metric_name_str, unit_pattern, "_mb"); + strlcpy(metric.name, metric_name_str.c_str(), METRIC_NAME_MAX); + metric.type = METRIC_VALUE_TYPE_D; + metric.unit = METRIC_VALUE_UNIT_MEMORY_MEGABYTES; + break; + } + case METRIC_VALUE_TYPE_U64: + { + metric.value.d = libs::metrics::convert_memory(metric.unit, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, metric.value.u64); + std::string metric_name_str(metric.name); + RE2::GlobalReplace(&metric_name_str, unit_pattern, "_mb"); + strlcpy(metric.name, metric_name_str.c_str(), METRIC_NAME_MAX); + metric.type = METRIC_VALUE_TYPE_D; + metric.unit = METRIC_VALUE_UNIT_MEMORY_MEGABYTES; + break; + } + default: + break; + } + break; + default: + break; + } +} + +std::string prometheus_metrics_converter::convert_metric_to_text(metrics_v2 metric, std::string_view prometheus_namespace, std::string_view prometheus_subsystem, std::map const_labels) +{ + // Create `prometheus_metric_name_fully_qualified` + std::string prometheus_metric_name_fully_qualified; + if (!prometheus_namespace.empty()) + { + prometheus_metric_name_fully_qualified += std::string(prometheus_namespace) + "_"; + } + if (!prometheus_subsystem.empty()) + { + prometheus_metric_name_fully_qualified += std::string(prometheus_subsystem) + "_"; + } + prometheus_metric_name_fully_qualified += std::string(metric.name) + "_"; + prometheus_metric_name_fully_qualified += std::string(metrics_unit_name_mappings_prometheus[metric.unit]); + + // Create the complete 3-lines text-based Prometheus exposition format https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md + std::string prometheus_text = "# HELP " + prometheus_metric_name_fully_qualified + " https://falco.org/docs/metrics/\n"; + prometheus_text += "# TYPE " + prometheus_metric_name_fully_qualified + " " + std::string(metrics_metric_type_name_mappings_prometheus[metric.metric_type]) + "\n"; + prometheus_text += prometheus_metric_name_fully_qualified; + prometheus_text += "{raw_name=\"" + std::string(metric.name) + "\"" ; + for (const auto& [key, value] : const_labels) + { + prometheus_text += "," + key + "=\"" + value + "\"" ; + } + prometheus_text += "} "; // white space at the end important! + switch (metric.type) + { + case METRIC_VALUE_TYPE_U32: + prometheus_text += std::to_string(metric.value.u32); + break; + case METRIC_VALUE_TYPE_S32: + prometheus_text += std::to_string(metric.value.s32); + break; + case METRIC_VALUE_TYPE_U64: + prometheus_text += std::to_string(metric.value.u64); + break; + case METRIC_VALUE_TYPE_S64: + prometheus_text += std::to_string(metric.value.s64); + break; + case METRIC_VALUE_TYPE_D: + prometheus_text += std::to_string(metric.value.d); + break; + case METRIC_VALUE_TYPE_F: + prometheus_text += std::to_string(metric.value.f); + break; + case METRIC_VALUE_TYPE_I: + prometheus_text += std::to_string(metric.value.i); + break; + default: + break; + } + + prometheus_text += "\n"; + return prometheus_text; +} + +std::string prometheus_metrics_converter::convert_metric_to_text(std::string_view metric_name, std::string_view prometheus_namespace, std::string_view prometheus_subsystem, std::map const_labels) +{ + // Create `prometheus_metric_name_fully_qualified` + std::string prometheus_metric_name_fully_qualified; + if (!prometheus_namespace.empty()) + { + prometheus_metric_name_fully_qualified += std::string(prometheus_namespace) + "_"; + } + if (!prometheus_subsystem.empty()) + { + prometheus_metric_name_fully_qualified += std::string(prometheus_subsystem) + "_"; + } + prometheus_metric_name_fully_qualified += std::string(metric_name) + "_info"; + + // Create the complete 3-lines text-based Prometheus exposition format https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md + std::string prometheus_text = "# HELP " + prometheus_metric_name_fully_qualified + " https://falco.org/docs/metrics/\n"; + prometheus_text += "# TYPE " + prometheus_metric_name_fully_qualified + " gauge\n"; + prometheus_text += prometheus_metric_name_fully_qualified; + prometheus_text += "{raw_name=\"" + std::string(metric_name) + "\"" ; + for (const auto& [key, value] : const_labels) + { + prometheus_text += "," + key + "=\"" + value + "\"" ; + } + prometheus_text += "} 1\n"; + return prometheus_text; +} + +void prometheus_metrics_converter::convert_metric_to_unit_convention(metrics_v2& metric) +{ + // todo + return; +} + +void libs_metrics_collector::get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, uint32_t &vsz, uint32_t &pss, uint64_t &host_memory_used, uint64_t &host_open_fds) { FILE* f; char filepath[512]; @@ -157,7 +323,7 @@ void metrics_collector::get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, } } fclose(f); - memory_used_host = mem_total - mem_free - mem_buff - mem_cache; + host_memory_used = mem_total - mem_free - mem_buff - mem_cache; /* * Get total number of allocated file descriptors (not all open files!) @@ -171,7 +337,7 @@ void metrics_collector::get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, ASSERT(false); return; } - int matched_fds = fscanf(f, "%" SCNu64, &open_fds_host); + int matched_fds = fscanf(f, "%" SCNu64, &host_open_fds); fclose(f); if (matched_fds != 1) { @@ -180,7 +346,7 @@ void metrics_collector::get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, } } -void metrics_collector::get_cpu_usage_and_total_procs(double start_time, double &cpu_usage_perc, double &cpu_usage_perc_total_host, uint32_t &procs_running_host) +void libs_metrics_collector::get_cpu_usage_and_total_procs(double start_time, double &cpu_usage_perc, double &host_cpu_usage_perc, uint32_t &host_procs_running) { FILE* f; char filepath[512]; @@ -265,7 +431,7 @@ void metrics_collector::get_cpu_usage_and_total_procs(double start_time, double } else if(strncmp(line, "procs_running ", 14) == 0) { - sscanf(line, "procs_running %" SCNu32, &procs_running_host); + sscanf(line, "procs_running %" SCNu32, &host_procs_running); break; } } @@ -273,12 +439,12 @@ void metrics_collector::get_cpu_usage_and_total_procs(double start_time, double auto sum = user + nice + system + idle + iowait + irq + softirq; if (sum > 0) { - cpu_usage_perc_total_host = 100.0 - ((idle * 100.0) / sum); - cpu_usage_perc_total_host = std::round(cpu_usage_perc_total_host * 10.0) / 10.0; // round to 1 decimal + host_cpu_usage_perc = 100.0 - ((idle * 100.0) / sum); + host_cpu_usage_perc = std::round(host_cpu_usage_perc * 10.0) / 10.0; // round to 1 decimal } } -uint64_t metrics_collector::get_container_memory_usage() const +uint64_t libs_metrics_collector::get_container_memory_used() const { /* In Kubernetes `container_memory_working_set_bytes` is the memory measure the OOM killer uses * and values from `/sys/fs/cgroup/memory/memory.usage_in_bytes` are close enough. @@ -310,7 +476,7 @@ uint64_t metrics_collector::get_container_memory_usage() const return memory_used; } -void metrics_collector::snapshot() +void libs_metrics_collector::snapshot() { m_metrics.clear(); if (!m_inspector) @@ -343,46 +509,77 @@ void metrics_collector::snapshot() if((m_metrics_flags & METRICS_V2_RESOURCE_UTILIZATION)) { const scap_agent_info* agent_info = m_inspector->get_agent_info(); - uint32_t rss{0}, vsz{0}, pss{0}, procs_running_host{0}; - uint64_t memory_used_host{0}, open_fds_host{0}; - double cpu_usage_perc{0.0}, cpu_usage_perc_total_host{0.0}; - uint64_t container_memory_usage = get_container_memory_usage(); - metrics_v2_value_unit rss_unit{METRIC_VALUE_UNIT_MEMORY_KIBIBYTES}, vsz_unit{METRIC_VALUE_UNIT_MEMORY_KIBIBYTES}, - pss_unit{METRIC_VALUE_UNIT_MEMORY_KIBIBYTES}, memory_used_host_unit{METRIC_VALUE_UNIT_MEMORY_KIBIBYTES}, container_memory_usage_unit{METRIC_VALUE_UNIT_MEMORY_BYTES}; - metrics_v2_value_type rss_type{METRIC_VALUE_TYPE_U32}, vsz_type{METRIC_VALUE_TYPE_U32}, - pss_type{METRIC_VALUE_TYPE_U32}, memory_used_host_type{METRIC_VALUE_TYPE_U32}, container_memory_usage_type{METRIC_VALUE_TYPE_U64}; - get_cpu_usage_and_total_procs(agent_info->start_time, cpu_usage_perc, cpu_usage_perc_total_host, procs_running_host); - get_rss_vsz_pss_total_memory_and_open_fds(rss, vsz, pss, memory_used_host, open_fds_host); - if(m_convert_memory_to_mb) - { - rss = convert_memory(rss_unit, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, rss); - vsz = convert_memory(vsz_unit, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, vsz); - pss = convert_memory(pss_unit, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, pss); - memory_used_host = convert_memory(memory_used_host_unit, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, memory_used_host); - container_memory_usage = convert_memory(container_memory_usage_unit, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, container_memory_usage); - rss_unit = vsz_unit = pss_unit = memory_used_host_unit = container_memory_usage_unit = METRIC_VALUE_UNIT_MEMORY_MEGABYTES; - rss_type = vsz_type = pss_type = memory_used_host_type = container_memory_usage_type = METRIC_VALUE_TYPE_D; - } + uint32_t rss{0}, vsz{0}, pss{0}, host_procs_running{0}; + uint64_t host_memory_used{0}, host_open_fds{0}; + double cpu_usage_perc{0.0}, host_cpu_usage_perc{0.0}; + uint64_t container_memory_used = get_container_memory_used(); + get_cpu_usage_and_total_procs(agent_info->start_time, cpu_usage_perc, host_cpu_usage_perc, host_procs_running); + get_rss_vsz_pss_total_memory_and_open_fds(rss, vsz, pss, host_memory_used, host_open_fds); + // Resource utilization of the agent itself m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_CPU_PERC], - METRICS_V2_RESOURCE_UTILIZATION, METRIC_VALUE_TYPE_D, METRIC_VALUE_UNIT_PERC, METRIC_VALUE_NON_MONOTONIC_CURRENT, cpu_usage_perc)); + METRICS_V2_RESOURCE_UTILIZATION, + METRIC_VALUE_TYPE_D, + METRIC_VALUE_UNIT_PERC, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + cpu_usage_perc)); + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS], - METRICS_V2_RESOURCE_UTILIZATION, rss_type, rss_unit, METRIC_VALUE_NON_MONOTONIC_CURRENT, rss)); + METRICS_V2_RESOURCE_UTILIZATION, + METRIC_VALUE_TYPE_U32, + METRIC_VALUE_UNIT_MEMORY_KIBIBYTES, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + rss)); + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ], - METRICS_V2_RESOURCE_UTILIZATION, vsz_type, vsz_unit, METRIC_VALUE_NON_MONOTONIC_CURRENT, vsz)); + METRICS_V2_RESOURCE_UTILIZATION, + METRIC_VALUE_TYPE_U32, + METRIC_VALUE_UNIT_MEMORY_KIBIBYTES, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + vsz)); + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS], - METRICS_V2_RESOURCE_UTILIZATION, pss_type, pss_unit, METRIC_VALUE_NON_MONOTONIC_CURRENT, pss)); + METRICS_V2_RESOURCE_UTILIZATION, + METRIC_VALUE_TYPE_U32, + METRIC_VALUE_UNIT_MEMORY_KIBIBYTES, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + pss)); + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY], - METRICS_V2_RESOURCE_UTILIZATION, container_memory_usage_type, container_memory_usage_unit, METRIC_VALUE_NON_MONOTONIC_CURRENT, container_memory_usage)); + METRICS_V2_RESOURCE_UTILIZATION, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_MEMORY_BYTES, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + container_memory_used)); + // Resource utilization / load indicators of the underlying host - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST], - METRICS_V2_RESOURCE_UTILIZATION, METRIC_VALUE_TYPE_D, METRIC_VALUE_UNIT_PERC, METRIC_VALUE_NON_MONOTONIC_CURRENT, cpu_usage_perc_total_host)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_PROCS_HOST], - METRICS_V2_RESOURCE_UTILIZATION, METRIC_VALUE_TYPE_U32, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_NON_MONOTONIC_CURRENT, procs_running_host)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST], - METRICS_V2_RESOURCE_UTILIZATION, memory_used_host_type, memory_used_host_unit, METRIC_VALUE_NON_MONOTONIC_CURRENT, memory_used_host)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST], - METRICS_V2_RESOURCE_UTILIZATION, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_NON_MONOTONIC_CURRENT, open_fds_host)); + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_HOST_CPU_PERC], + METRICS_V2_RESOURCE_UTILIZATION, + METRIC_VALUE_TYPE_D, + METRIC_VALUE_UNIT_PERC, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + host_cpu_usage_perc)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_HOST_PROCS], + METRICS_V2_RESOURCE_UTILIZATION, + METRIC_VALUE_TYPE_U32, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + host_procs_running)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_HOST_MEMORY], + METRICS_V2_RESOURCE_UTILIZATION, + METRIC_VALUE_TYPE_U32, + METRIC_VALUE_UNIT_MEMORY_KIBIBYTES, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + host_memory_used)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_HOST_FDS], + METRICS_V2_RESOURCE_UTILIZATION, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + host_open_fds)); } if((m_metrics_flags & METRICS_V2_STATE_COUNTERS)) @@ -417,143 +614,157 @@ void metrics_collector::snapshot() } } - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_THREADS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_NON_MONOTONIC_CURRENT, n_threads)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_FDS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_NON_MONOTONIC_CURRENT, n_fds)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_noncached_fd_lookups)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_CACHED_FD_LOOKUPS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_cached_fd_lookups)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_FAILED_FD_LOOKUPS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_failed_fd_lookups)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_ADDED_FDS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_added_fds)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_REMOVED_FDS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_removed_fds)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_STORED_EVTS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_stored_evts)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_STORE_EVTS_DROPS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_store_evts_drops)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_RETRIEVED_EVTS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_retrieved_evts)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_retrieve_evts_drops)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_noncached_thread_lookups)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_cached_thread_lookups)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_failed_thread_lookups)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_ADDED_THREADS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_added_threads)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_REMOVED_THREADS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U64, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_removed_threads)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U32, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_MONOTONIC, sinsp_stats_v2->m_n_drops_full_threadtable)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_MISSING_CONTAINER_IMAGES], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U32, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_NON_MONOTONIC_CURRENT, sinsp_stats_v2->m_n_missing_container_images)); - m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_CONTAINERS], \ - METRICS_V2_STATE_COUNTERS, METRIC_VALUE_TYPE_U32, METRIC_VALUE_UNIT_COUNT, METRIC_VALUE_NON_MONOTONIC_CURRENT, sinsp_stats_v2->m_n_containers)); + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_THREADS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + n_threads)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_FDS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + n_fds)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_noncached_fd_lookups)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_CACHED_FD_LOOKUPS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_cached_fd_lookups)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_FAILED_FD_LOOKUPS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_failed_fd_lookups)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_ADDED_FDS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_added_fds)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_REMOVED_FDS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_removed_fds)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_STORED_EVTS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_stored_evts)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_STORE_EVTS_DROPS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_store_evts_drops)); + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_RETRIEVED_EVTS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_retrieved_evts)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_retrieve_evts_drops)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_noncached_thread_lookups)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_cached_thread_lookups)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_failed_thread_lookups)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_ADDED_THREADS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_added_threads)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_REMOVED_THREADS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U64, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_removed_threads)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U32, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_MONOTONIC, + sinsp_stats_v2->m_n_drops_full_threadtable)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_MISSING_CONTAINER_IMAGES], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U32, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + sinsp_stats_v2->m_n_missing_container_images)); + + m_metrics.emplace_back(new_metric(sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_CONTAINERS], + METRICS_V2_STATE_COUNTERS, + METRIC_VALUE_TYPE_U32, + METRIC_VALUE_UNIT_COUNT, + METRIC_VALUE_NON_MONOTONIC_CURRENT, + sinsp_stats_v2->m_n_containers)); } } } -std::string metrics_collector::convert_metric_to_prometheus_text(metrics_v2 metric, std::string_view prometheus_namespace, std::string_view prometheus_subsystem, std::map const_labels) -{ - // Create `prometheus_metric_name_fully_qualified` - std::string prometheus_metric_name_fully_qualified; - if (!prometheus_namespace.empty()) - { - prometheus_metric_name_fully_qualified += std::string(prometheus_namespace) + "_"; - } - if (!prometheus_subsystem.empty()) - { - prometheus_metric_name_fully_qualified += std::string(prometheus_subsystem) + "_"; - } - prometheus_metric_name_fully_qualified += std::string(metric.name) + "_"; - prometheus_metric_name_fully_qualified += std::string(metrics_unit_name_mappings_prometheus[metric.unit]); - - // Create the complete 3-lines text-based Prometheus exposition format https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md - std::string prometheus_text = "# HELP " + prometheus_metric_name_fully_qualified + " https://falco.org/docs/metrics/\n"; - prometheus_text += "# TYPE " + prometheus_metric_name_fully_qualified + " " + std::string(metrics_metric_type_name_mappings_prometheus[metric.metric_type]) + "\n"; - prometheus_text += prometheus_metric_name_fully_qualified; - prometheus_text += "{raw_name=\"" + std::string(metric.name) + "\"" ; - for (const auto& [key, value] : const_labels) - { - prometheus_text += "," + key + "=\"" + value + "\"" ; - } - prometheus_text += "} "; // white space at the end important! - switch (metric.type) - { - case METRIC_VALUE_TYPE_U32: - prometheus_text += std::to_string(metric.value.u32); - break; - case METRIC_VALUE_TYPE_S32: - prometheus_text += std::to_string(metric.value.s32); - break; - case METRIC_VALUE_TYPE_U64: - prometheus_text += std::to_string(metric.value.u64); - break; - case METRIC_VALUE_TYPE_S64: - prometheus_text += std::to_string(metric.value.s64); - break; - case METRIC_VALUE_TYPE_D: - prometheus_text += std::to_string(metric.value.d); - break; - case METRIC_VALUE_TYPE_F: - prometheus_text += std::to_string(metric.value.f); - break; - case METRIC_VALUE_TYPE_I: - prometheus_text += std::to_string(metric.value.i); - break; - default: - break; - } - - prometheus_text += "\n"; - return prometheus_text; -} - -std::string metrics_collector::convert_metric_to_prometheus_text(std::string_view metric_name, std::string_view prometheus_namespace, std::string_view prometheus_subsystem, std::map const_labels) +const std::vector& libs_metrics_collector::get_metrics() const { - // Create `prometheus_metric_name_fully_qualified` - std::string prometheus_metric_name_fully_qualified; - if (!prometheus_namespace.empty()) - { - prometheus_metric_name_fully_qualified += std::string(prometheus_namespace) + "_"; - } - if (!prometheus_subsystem.empty()) - { - prometheus_metric_name_fully_qualified += std::string(prometheus_subsystem) + "_"; - } - prometheus_metric_name_fully_qualified += std::string(metric_name) + "_info"; - - // Create the complete 3-lines text-based Prometheus exposition format https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md - std::string prometheus_text = "# HELP " + prometheus_metric_name_fully_qualified + " https://falco.org/docs/metrics/\n"; - prometheus_text += "# TYPE " + prometheus_metric_name_fully_qualified + " gauge\n"; - prometheus_text += prometheus_metric_name_fully_qualified; - prometheus_text += "{raw_name=\"" + std::string(metric_name) + "\"" ; - for (const auto& [key, value] : const_labels) - { - prometheus_text += "," + key + "=\"" + value + "\"" ; - } - prometheus_text += "} 1\n"; - return prometheus_text; + return m_metrics; } -const std::vector& metrics_collector::get_metrics() const +std::vector& libs_metrics_collector::get_metrics() { return m_metrics; } -metrics_collector::metrics_collector(sinsp* inspector, uint32_t flags, bool convert_memory_to_mb) : +libs_metrics_collector::libs_metrics_collector(sinsp* inspector, uint32_t flags) : m_inspector(inspector), - m_metrics_flags(flags), - m_convert_memory_to_mb(convert_memory_to_mb) + m_metrics_flags(flags) { } -} // namespace libsinsp::metrics +} // namespace libs::metrics #endif diff --git a/userspace/libsinsp/metrics_collector.h b/userspace/libsinsp/metrics_collector.h index 56aa30472e..7bd769c084 100644 --- a/userspace/libsinsp/metrics_collector.h +++ b/userspace/libsinsp/metrics_collector.h @@ -53,10 +53,10 @@ enum sinsp_stats_v2_resource_utilization SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ, ///< Current VSZ (Virtual Memory Size), calculated based on /proc/self/status info, unit: kb. SINSP_RESOURCE_UTILIZATION_MEMORY_PSS, ///< Current PSS (Proportional Set Size), calculated based on /proc/self/smaps_rollup info, unit: kb. SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY, ///< Cgroup current memory used, default Kubernetes /sys/fs/cgroup/memory/memory.usage_in_bytes, unit: bytes. - SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST, ///< Current total host CPU usage (all CPUs), calculated based on ${HOST_ROOT}/proc/stat info, unit: percentage. - SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST, ///< Current total memory used out of available host memory, calculated based on ${HOST_ROOT}/proc/meminfo info, unit: kb. - SINSP_RESOURCE_UTILIZATION_PROCS_HOST, ///< Number of processes currently running on CPUs on the host, retrieved from ${HOST_ROOT}/proc/stat line `procs_running`, unit: count. - SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST, ///< Number of allocated fds on the host, retrieved from ${HOST_ROOT}/proc/sys/fs/file-nr, unit: count. + SINSP_RESOURCE_UTILIZATION_HOST_CPU_PERC, ///< Current total host CPU usage (all CPUs), calculated based on ${HOST_ROOT}/proc/stat info, unit: percentage. + SINSP_RESOURCE_UTILIZATION_HOST_MEMORY, ///< Current total memory used out of available host memory, calculated based on ${HOST_ROOT}/proc/meminfo info, unit: kb. + SINSP_RESOURCE_UTILIZATION_HOST_PROCS, ///< Number of processes currently running on CPUs on the host, retrieved from ${HOST_ROOT}/proc/stat line `procs_running`, unit: count. + SINSP_RESOURCE_UTILIZATION_HOST_FDS, ///< Number of allocated fds on the host, retrieved from ${HOST_ROOT}/proc/sys/fs/file-nr, unit: count. SINSP_STATS_V2_N_THREADS, ///< Total number of threads currently stored in the sinsp state thread table, unit: count. SINSP_STATS_V2_N_FDS, ///< Total number of fds currently stored across all threadtables associated with each active thread in the sinsp state thread table, unit: count. SINSP_STATS_V2_NONCACHED_FD_LOOKUPS, ///< fdtable state related counters, unit: count. @@ -81,22 +81,58 @@ enum sinsp_stats_v2_resource_utilization #ifdef __linux__ -namespace libsinsp::metrics { +namespace libs::metrics +{ -class metrics_collector +template +static double convert_memory(metrics_v2_value_unit source_unit, metrics_v2_value_unit dest_unit, T val) +{ + double factor = 1; + switch(source_unit) + { + case METRIC_VALUE_UNIT_MEMORY_BYTES: + factor = 1; + break; + case METRIC_VALUE_UNIT_MEMORY_KIBIBYTES: + factor = 1024.; + break; + case METRIC_VALUE_UNIT_MEMORY_MEGABYTES: + factor = 1024. * 1024.; + break; + default: + return 0; + } + + double bytes_val = val * factor; + switch(dest_unit) + { + case METRIC_VALUE_UNIT_MEMORY_BYTES: + return bytes_val; + case METRIC_VALUE_UNIT_MEMORY_KIBIBYTES: + return std::round((bytes_val / 1024.) * 10.) / 10.; // round to 1 decimal + case METRIC_VALUE_UNIT_MEMORY_MEGABYTES: + return std::round((bytes_val / 1024. / 1024.) * 10.) / 10.; // round to 1 decimal + default: + return 0; + } + return 0; +} + +class metrics_converter { public: - metrics_collector(sinsp* inspector, uint32_t flags, bool convert_memory_to_mb); + metrics_converter() = default; - /*! - \brief Method to fill up m_metrics_buffer with metrics; refreshes m_metrics with up-to-date metrics on each call - */ - void snapshot(); + virtual std::string convert_metric_to_text(metrics_v2 metric); - /*! - \brief Method to get a const reference to m_metrics vector - */ - const std::vector& get_metrics() const; + virtual void convert_metric_to_unit_convention(metrics_v2& metric) = 0; +}; + +// Subclass for Prometheus-specific metric conversion +class prometheus_metrics_converter : public metrics_converter +{ +public: + prometheus_metrics_converter() = default; /*! \brief Method to convert a metrics_v2 metric to the text-based Prometheus exposition format. @@ -140,7 +176,7 @@ class metrics_collector * w/ a `prometheus_metric_name_fully_qualified` - optional components prepended to and unit appended to. * 3-lines including # HELP and # TYPE lines followed by the metric line, raw metric name always present as label. */ - std::string convert_metric_to_prometheus_text(metrics_v2 metric, std::string_view prometheus_namespace = "", std::string_view prometheus_subsystem = "", std::map const_labels = {}); + std::string convert_metric_to_text(metrics_v2 metric, std::string_view prometheus_namespace = "", std::string_view prometheus_subsystem = "", std::map const_labels = {}); /*! \brief Method to convert a software version like metric_name to the text-based Prometheus exposition format. @@ -165,44 +201,47 @@ class metrics_collector * w/ a `prometheus_metric_name_fully_qualified` - optional components prepended to and unit appended to. * 3-lines including # HELP and # TYPE lines followed by the metric line, raw metric name always present as label. */ - std::string convert_metric_to_prometheus_text(std::string_view metric_name, std::string_view prometheus_namespace = "", std::string_view prometheus_subsystem = "", std::map const_labels = {}); + std::string convert_metric_to_text(std::string_view metric_name, std::string_view prometheus_namespace = "", std::string_view prometheus_subsystem = "", std::map const_labels = {}); /*! - \brief Method to convert memory units; tied to metrics_v2 definitions + \brief Method to convert metric units to Prometheus base units. todo, not yet implemented. + * */ - template - static double convert_memory(metrics_v2_value_unit source_unit, metrics_v2_value_unit dest_unit, T val) - { - double factor = 1; - switch(source_unit) - { - case METRIC_VALUE_UNIT_MEMORY_BYTES: - factor = 1; - break; - case METRIC_VALUE_UNIT_MEMORY_KIBIBYTES: - factor = 1024.; - break; - case METRIC_VALUE_UNIT_MEMORY_MEGABYTES: - factor = 1024. * 1024.; - break; - default: - return 0; - } + void convert_metric_to_unit_convention(metrics_v2& metric) override; +}; - double bytes_val = val * factor; - switch(dest_unit) - { - case METRIC_VALUE_UNIT_MEMORY_BYTES: - return bytes_val; - case METRIC_VALUE_UNIT_MEMORY_KIBIBYTES: - return std::round((bytes_val / 1024.) * 10.) / 10.; // round to 1 decimal - case METRIC_VALUE_UNIT_MEMORY_MEGABYTES: - return std::round((bytes_val / 1024. / 1024.) * 10.) / 10.; // round to 1 decimal - default: - return 0; - } - return 0; - } +// Subclass for output_rule-specific metric conversion +class output_rule_metrics_converter : public metrics_converter +{ +public: + output_rule_metrics_converter() = default; + + /*! + \brief Method to convert metric units of memory-related metrics to mb + * + */ + void convert_metric_to_unit_convention(metrics_v2& metric) override; +}; + +class libs_metrics_collector +{ +public: + libs_metrics_collector(sinsp* inspector, uint32_t flags); + + /*! + \brief Method to fill up m_metrics_buffer with metrics; refreshes m_metrics with up-to-date metrics on each call + */ + void snapshot(); + + /*! + \brief Method to get a const reference to m_metrics vector + */ + const std::vector& get_metrics() const; + + /*! + \brief Method to get a non-const reference to m_metrics vector + */ + std::vector& get_metrics(); /*! \brief Method to create a new metrics_v2 @@ -216,22 +255,21 @@ class metrics_collector metric.type = type; metric.unit = unit; metric.metric_type = metric_type; - set_new_metric(metric, type, val); + set_metric_value(metric, type, val); return metric; } private: sinsp* m_inspector; uint32_t m_metrics_flags = METRICS_V2_KERNEL_COUNTERS | METRICS_V2_LIBBPF_STATS | METRICS_V2_RESOURCE_UTILIZATION | METRICS_V2_STATE_COUNTERS; - bool m_convert_memory_to_mb = true; std::vector m_metrics; - void get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, uint32_t &vsz, uint32_t &pss, uint64_t &memory_used_host, uint64_t &open_fds_host); - void get_cpu_usage_and_total_procs(double start_time, double &cpu_usage_perc, double &cpu_usage_perc_total_host, uint32_t &procs_running_host); - uint64_t get_container_memory_usage() const; + void get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, uint32_t &vsz, uint32_t &pss, uint64_t &host_memory_used, uint64_t &host_open_fds); + void get_cpu_usage_and_total_procs(double start_time, double &cpu_usage_perc, double &host_cpu_usage_perc, uint32_t &host_procs_running); + uint64_t get_container_memory_used() const; template - static void set_new_metric(metrics_v2& metric, metrics_v2_value_type type, T val) + static void set_metric_value(metrics_v2& metric, metrics_v2_value_type type, T val) { switch (type) { @@ -262,6 +300,6 @@ class metrics_collector } }; -} // namespace libsinsp::metrics +} // namespace libs::metrics #endif diff --git a/userspace/libsinsp/test/sinsp_metrics.ut.cpp b/userspace/libsinsp/test/sinsp_metrics.ut.cpp index 585ded7a60..e93afb96cf 100644 --- a/userspace/libsinsp/test/sinsp_metrics.ut.cpp +++ b/userspace/libsinsp/test/sinsp_metrics.ut.cpp @@ -21,7 +21,7 @@ limitations under the License. #include "sinsp_with_test_input.h" #include -TEST_F(sinsp_with_test_input, sinsp_metrics_collector) +TEST_F(sinsp_with_test_input, sinsp_libs_metrics_collector_prometheus) { m_inspector.set_sinsp_stats_v2_enabled(); // Extra call to verify that we don't fail @@ -32,18 +32,106 @@ TEST_F(sinsp_with_test_input, sinsp_metrics_collector) /* Snapshot current metrics and get the updated metrics_snapshot buffer */ uint32_t test_metrics_flags = (METRICS_V2_KERNEL_COUNTERS | METRICS_V2_LIBBPF_STATS | METRICS_V2_RESOURCE_UTILIZATION | METRICS_V2_STATE_COUNTERS); - bool convert_memory_to_mb = true; - auto metrics_collector = std::make_unique(&m_inspector, test_metrics_flags, convert_memory_to_mb); - metrics_collector->snapshot(); - auto metrics_snapshot = metrics_collector->get_metrics(); + auto libs_metrics_collector = std::make_unique(&m_inspector, test_metrics_flags); + auto prometheus_metrics_converter = std::make_unique(); + + libs_metrics_collector->snapshot(); + auto metrics_snapshot = libs_metrics_collector->get_metrics(); + ASSERT_EQ(metrics_snapshot.size(), 28); + + /* Test prometheus_metrics_converter->convert_metric_to_text */ + for (const auto& metric: metrics_snapshot) + { + if (strncmp(metric.name, "n_missing_container_images", 17) == 0) + { + // This resembles the Falco client use case + + // Falco output_rule metrics prepends either `falco.` or `scap.` to a single metric, see https://falco.org/docs/metrics/ + // Use same strings for `prometheus_subsystem`, but instead of `.` we use `_` delimiter to conform with Prometheus naming conventions + append the unit + std::string prometheus_text = prometheus_metrics_converter->convert_metric_to_text(metric, "testns", "falco", {{"example_key1", "example1"},{"example_key2", "example2"}}); + std::string prometheus_text_substring = R"(# HELP testns_falco_n_missing_container_images_total https://falco.org/docs/metrics/ +# TYPE testns_falco_n_missing_container_images_total gauge +testns_falco_n_missing_container_images_total{raw_name="n_missing_container_images",example_key1="example1",example_key2="example2"} 0 +)"; + ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; + ASSERT_GT(prometheus_text.length(), 8); + // Test only one const_labels + prometheus_text = prometheus_metrics_converter->convert_metric_to_text(metric, "testns", "falco", {{"example_key1", "example1"}}); + prometheus_text_substring = R"(# HELP testns_falco_n_missing_container_images_total https://falco.org/docs/metrics/ +# TYPE testns_falco_n_missing_container_images_total gauge +testns_falco_n_missing_container_images_total{raw_name="n_missing_container_images",example_key1="example1"} 0 +)"; + ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; + ASSERT_GT(prometheus_text.length(), 8); + // Test no const_labels + prometheus_text = prometheus_metrics_converter->convert_metric_to_text(metric, "testns", "falco"); + prometheus_text_substring = R"(# HELP testns_falco_n_missing_container_images_total https://falco.org/docs/metrics/ +# TYPE testns_falco_n_missing_container_images_total gauge +testns_falco_n_missing_container_images_total{raw_name="n_missing_container_images"} 0 +)"; + ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; + ASSERT_GT(prometheus_text.length(), 8); + // Test no prometheus_subsytem + prometheus_text = prometheus_metrics_converter->convert_metric_to_text(metric, "testns"); + prometheus_text_substring = R"(# HELP testns_n_missing_container_images_total https://falco.org/docs/metrics/ +# TYPE testns_n_missing_container_images_total gauge +testns_n_missing_container_images_total{raw_name="n_missing_container_images"} 0 +)"; + ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; + ASSERT_GT(prometheus_text.length(), 8); + // Test no prometheus_namespace + prometheus_text = prometheus_metrics_converter->convert_metric_to_text(metric); + prometheus_text_substring = R"(# HELP n_missing_container_images_total https://falco.org/docs/metrics/ +# TYPE n_missing_container_images_total gauge +n_missing_container_images_total{raw_name="n_missing_container_images"} 0 +)"; + ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; + ASSERT_GT(prometheus_text.length(), 8); + // Test no prometheus_namespace, but prometheus_subsytem + prometheus_text = prometheus_metrics_converter->convert_metric_to_text(metric, "", "falco"); + prometheus_text_substring = R"(# HELP falco_n_missing_container_images_total https://falco.org/docs/metrics/ +# TYPE falco_n_missing_container_images_total gauge +falco_n_missing_container_images_total{raw_name="n_missing_container_images"} 0 +)"; + ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; + ASSERT_GT(prometheus_text.length(), 8); + } + } + + std::string prometheus_text = prometheus_metrics_converter->convert_metric_to_text("kernel_release", "testns", "falco", {{"kernel_release", "6.6.7-200.fc39.x86_64"}}); + std::string prometheus_text_substring = R"(# HELP testns_falco_kernel_release_info https://falco.org/docs/metrics/ +# TYPE testns_falco_kernel_release_info gauge +testns_falco_kernel_release_info{raw_name="kernel_release",kernel_release="6.6.7-200.fc39.x86_64"} 1 +)"; + ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; + ASSERT_GT(prometheus_text.length(), 8); + +} + +TEST_F(sinsp_with_test_input, sinsp_libs_metrics_collector_output_rule) +{ + m_inspector.set_sinsp_stats_v2_enabled(); + // Extra call to verify that we don't fail + m_inspector.set_sinsp_stats_v2_enabled(); + DEFAULT_TREE + auto evt = generate_random_event(p2_t1_tid); + ASSERT_EQ(get_field_as_string(evt, "proc.nthreads"), "3"); + + /* Snapshot current metrics and get the updated metrics_snapshot buffer */ + uint32_t test_metrics_flags = (METRICS_V2_KERNEL_COUNTERS | METRICS_V2_LIBBPF_STATS | METRICS_V2_RESOURCE_UTILIZATION | METRICS_V2_STATE_COUNTERS); + auto libs_metrics_collector = std::make_unique(&m_inspector, test_metrics_flags); + auto output_rule_metrics_converter = std::make_unique(); + /* Multiple calls */ - metrics_collector->snapshot(); - metrics_collector->snapshot(); - metrics_snapshot = metrics_collector->get_metrics(); + libs_metrics_collector->snapshot(); + auto metrics_snapshot = libs_metrics_collector->get_metrics(); + libs_metrics_collector->snapshot(); + libs_metrics_collector->snapshot(); + metrics_snapshot = libs_metrics_collector->get_metrics(); ASSERT_EQ(metrics_snapshot.size(), 28); /* These names should always be available, note that we currently can't check for the merged scap stats metrics here */ - std::unordered_set minimal_metrics_names = {"cpu_usage_perc", "memory_rss", "open_fds_host", \ + std::unordered_set minimal_metrics_names = {"cpu_usage_perc", "memory_rss_kb", "host_open_fds", \ "n_threads", "n_fds", "n_added_fds", "n_added_threads", "n_removed_threads", "n_containers"}; for(const auto& metric_name : minimal_metrics_names) @@ -64,17 +152,19 @@ TEST_F(sinsp_with_test_input, sinsp_metrics_collector) } /* Assert successful memory unit changes and sanity check some values to be greater than 0 */ - const std::vector metrics_names_memory = {"memory_rss", "memory_vsz", "memory_pss", "container_memory_used", "memory_used_host"}; + const std::vector metrics_names_memory = {"memory_rss_mb", "memory_vsz_mb", "memory_pss_mb", "container_memory_used_mb", "host_memory_used_mb"}; const std::vector metrics_names_values_gt = {"n_threads", "n_fds", "n_added_threads"}; uint32_t success_memory_cnt = 0; uint32_t success_values_cnt = 0; - for (const auto& metric: metrics_snapshot) + for (auto& metric: metrics_snapshot) { + // This resembles the Falco client use case and would be called if `convert_memory_to_mb` is set to true + output_rule_metrics_converter->convert_metric_to_unit_convention(metric); if (std::find(metrics_names_memory.begin(), metrics_names_memory.end(), metric.name) != metrics_names_memory.end()) { ASSERT_EQ(metric.unit, METRIC_VALUE_UNIT_MEMORY_MEGABYTES); ASSERT_EQ(metric.type, METRIC_VALUE_TYPE_D); - if (strncmp(metric.name, "memory_used_host", 17) == 0 || strncmp(metric.name, "memory_rss", 11) == 0 ) + if (strncmp(metric.name, "host_memory_used_mb", 20) == 0 || strncmp(metric.name, "memory_rss_mb", 14) == 0 ) { ASSERT_GT(metric.value.d, 0); // Just making sure we don't get a high value due to an unitialized variables @@ -97,121 +187,57 @@ TEST_F(sinsp_with_test_input, sinsp_metrics_collector) ASSERT_EQ(success_values_cnt, metrics_names_values_gt.size()); /* Empty call */ - metrics_collector = std::make_unique(&m_inspector, 0, convert_memory_to_mb); - metrics_collector->snapshot(); - metrics_snapshot = metrics_collector->get_metrics(); + libs_metrics_collector = std::make_unique(&m_inspector, 0); + libs_metrics_collector->snapshot(); + metrics_snapshot = libs_metrics_collector->get_metrics(); ASSERT_TRUE(metrics_snapshot.empty()); /* Sanity check empty inspector */ test_metrics_flags = (METRICS_V2_RESOURCE_UTILIZATION | METRICS_V2_STATE_COUNTERS); - metrics_collector = std::make_unique(nullptr, test_metrics_flags, convert_memory_to_mb); - metrics_collector->snapshot(); - metrics_snapshot = metrics_collector->get_metrics(); + libs_metrics_collector = std::make_unique(nullptr, test_metrics_flags); + libs_metrics_collector->snapshot(); + metrics_snapshot = libs_metrics_collector->get_metrics(); ASSERT_TRUE(metrics_snapshot.empty()); /* Some sanity checks for selective flags */ test_metrics_flags = 0; test_metrics_flags |= METRICS_V2_KERNEL_COUNTERS; // 20, but can't test it here it's 0 test_metrics_flags |= METRICS_V2_LIBBPF_STATS; // 21 (x86_64 machine), but can't test it here it's 0 - metrics_collector = std::make_unique(&m_inspector, test_metrics_flags, convert_memory_to_mb); - metrics_collector->snapshot(); - metrics_snapshot = metrics_collector->get_metrics(); + libs_metrics_collector = std::make_unique(&m_inspector, test_metrics_flags); + libs_metrics_collector->snapshot(); + metrics_snapshot = libs_metrics_collector->get_metrics(); ASSERT_TRUE(metrics_snapshot.empty()); test_metrics_flags = 0; test_metrics_flags |= METRICS_V2_RESOURCE_UTILIZATION; - metrics_collector = std::make_unique(&m_inspector, test_metrics_flags, convert_memory_to_mb); - metrics_collector->snapshot(); - metrics_snapshot = metrics_collector->get_metrics(); + libs_metrics_collector = std::make_unique(&m_inspector, test_metrics_flags); + libs_metrics_collector->snapshot(); + metrics_snapshot = libs_metrics_collector->get_metrics(); ASSERT_EQ(metrics_snapshot.size(), 9); test_metrics_flags = 0; test_metrics_flags |= METRICS_V2_STATE_COUNTERS; - metrics_collector = std::make_unique(&m_inspector, test_metrics_flags, convert_memory_to_mb); - metrics_collector->snapshot(); - metrics_snapshot = metrics_collector->get_metrics(); + libs_metrics_collector = std::make_unique(&m_inspector, test_metrics_flags); + libs_metrics_collector->snapshot(); + metrics_snapshot = libs_metrics_collector->get_metrics(); ASSERT_EQ(metrics_snapshot.size(), 19); test_metrics_flags = (METRICS_V2_RESOURCE_UTILIZATION | METRICS_V2_STATE_COUNTERS); - metrics_collector = std::make_unique(&m_inspector, test_metrics_flags, convert_memory_to_mb); - metrics_collector->snapshot(); - metrics_snapshot = metrics_collector->get_metrics(); + libs_metrics_collector = std::make_unique(&m_inspector, test_metrics_flags); + libs_metrics_collector->snapshot(); + metrics_snapshot = libs_metrics_collector->get_metrics(); ASSERT_EQ(metrics_snapshot.size(), 28); +} - /* Test public convert_memory method */ - double converted_memory = metrics_collector->convert_memory(METRIC_VALUE_UNIT_MEMORY_BYTES, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, (uint64_t)52428800); +TEST(sinsp_libs_metrics, sinsp_libs_metrics_convert_units) +{ + /* Test public libs::metrics::convert_memory method */ + double converted_memory = libs::metrics::convert_memory(METRIC_VALUE_UNIT_MEMORY_BYTES, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, (uint64_t)52428800); ASSERT_EQ(converted_memory, 50); - converted_memory = metrics_collector->convert_memory(METRIC_VALUE_UNIT_MEMORY_KIBIBYTES, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, (uint64_t)51200); + converted_memory = libs::metrics::convert_memory(METRIC_VALUE_UNIT_MEMORY_KIBIBYTES, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, (uint64_t)51200); ASSERT_EQ(converted_memory, 50); - converted_memory = metrics_collector->convert_memory(METRIC_VALUE_UNIT_MEMORY_MEGABYTES, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, (uint64_t)50); + converted_memory = libs::metrics::convert_memory(METRIC_VALUE_UNIT_MEMORY_MEGABYTES, METRIC_VALUE_UNIT_MEMORY_MEGABYTES, (uint64_t)50); ASSERT_EQ(converted_memory, 50); - - /* Test public convert_metric_to_prometheus_text */ - - for (const auto& metric: metrics_snapshot) - { - if (strncmp(metric.name, "n_missing_container_images", 17) == 0) - { - // This resembles the Falco client use case - - // Falco output_rule metrics prepends either `falco.` or `scap.` to a single metric, see https://falco.org/docs/metrics/ - // Use same strings for `prometheus_subsystem`, but instead of `.` we use `_` delimiter to conform with Prometheus naming conventions + append the unit - std::string prometheus_text = metrics_collector->convert_metric_to_prometheus_text(metric, "testns", "falco", {{"example_key1", "example1"},{"example_key2", "example2"}}); - std::string prometheus_text_substring = R"(# HELP testns_falco_n_missing_container_images_total https://falco.org/docs/metrics/ -# TYPE testns_falco_n_missing_container_images_total gauge -testns_falco_n_missing_container_images_total{raw_name="n_missing_container_images",example_key1="example1",example_key2="example2"} 0 -)"; - ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; - ASSERT_GT(prometheus_text.length(), 8); - // Test only one const_labels - prometheus_text = metrics_collector->convert_metric_to_prometheus_text(metric, "testns", "falco", {{"example_key1", "example1"}}); - prometheus_text_substring = R"(# HELP testns_falco_n_missing_container_images_total https://falco.org/docs/metrics/ -# TYPE testns_falco_n_missing_container_images_total gauge -testns_falco_n_missing_container_images_total{raw_name="n_missing_container_images",example_key1="example1"} 0 -)"; - ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; - ASSERT_GT(prometheus_text.length(), 8); - // Test no const_labels - prometheus_text = metrics_collector->convert_metric_to_prometheus_text(metric, "testns", "falco"); - prometheus_text_substring = R"(# HELP testns_falco_n_missing_container_images_total https://falco.org/docs/metrics/ -# TYPE testns_falco_n_missing_container_images_total gauge -testns_falco_n_missing_container_images_total{raw_name="n_missing_container_images"} 0 -)"; - ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; - ASSERT_GT(prometheus_text.length(), 8); - // Test no prometheus_subsytem - prometheus_text = metrics_collector->convert_metric_to_prometheus_text(metric, "testns"); - prometheus_text_substring = R"(# HELP testns_n_missing_container_images_total https://falco.org/docs/metrics/ -# TYPE testns_n_missing_container_images_total gauge -testns_n_missing_container_images_total{raw_name="n_missing_container_images"} 0 -)"; - ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; - ASSERT_GT(prometheus_text.length(), 8); - // Test no prometheus_namespace - prometheus_text = metrics_collector->convert_metric_to_prometheus_text(metric); - prometheus_text_substring = R"(# HELP n_missing_container_images_total https://falco.org/docs/metrics/ -# TYPE n_missing_container_images_total gauge -n_missing_container_images_total{raw_name="n_missing_container_images"} 0 -)"; - ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; - ASSERT_GT(prometheus_text.length(), 8); - // Test no prometheus_namespace, but prometheus_subsytem - prometheus_text = metrics_collector->convert_metric_to_prometheus_text(metric, "", "falco"); - prometheus_text_substring = R"(# HELP falco_n_missing_container_images_total https://falco.org/docs/metrics/ -# TYPE falco_n_missing_container_images_total gauge -falco_n_missing_container_images_total{raw_name="n_missing_container_images"} 0 -)"; - ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; - ASSERT_GT(prometheus_text.length(), 8); - } - } - - std::string prometheus_text = metrics_collector->convert_metric_to_prometheus_text("kernel_release", "testns", "falco", {{"kernel_release", "6.6.7-200.fc39.x86_64"}}); - std::string prometheus_text_substring = R"(# HELP testns_falco_kernel_release_info https://falco.org/docs/metrics/ -# TYPE testns_falco_kernel_release_info gauge -testns_falco_kernel_release_info{raw_name="kernel_release",kernel_release="6.6.7-200.fc39.x86_64"} 1 -)"; - ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text"; - ASSERT_GT(prometheus_text.length(), 8); } + #endif