From 6d7bfcd8d0e16b2b8305a380c4ae719cde77939b Mon Sep 17 00:00:00 2001 From: mphanias <121247041+mphanias@users.noreply.github.com> Date: Thu, 24 Aug 2023 23:34:45 +0530 Subject: [PATCH] OM-92 & OM-94 (#92) added support for index-pressure added support for stats and configs at XDR-DC and XDR-DC-Namespace level moved prometheus push logic to a common function --- common.go | 47 ++- gauge_stats.go | 1 - gauge_stats_list.toml | 29 +- gauge_stats_test.go | 21 +- helper_test.go | 47 ++- mock_data_generator_test.go | 612 +++++++++++++++++++++++++----------- observer.go | 19 +- tests/mock_test_data.txt | 56 ++++ watcher_latency.go | 3 +- watcher_latency_test.go | 46 +-- watcher_namespaces.go | 306 ++++++++++++++---- watcher_namespaces_test.go | 102 +++--- watcher_node_stats.go | 49 ++- watcher_node_stats_test.go | 38 ++- watcher_sets.go | 19 +- watcher_sets_test.go | 44 +-- watcher_sindex.go | 21 +- watcher_sindex_test.go | 40 +-- watcher_users_test.go | 29 +- watcher_xdr.go | 122 +++++-- watcher_xdr_test.go | 91 +++--- 21 files changed, 1191 insertions(+), 551 deletions(-) create mode 100644 tests/mock_test_data.txt diff --git a/common.go b/common.go index 9f436c7f..d7bfb9cd 100644 --- a/common.go +++ b/common.go @@ -17,6 +17,7 @@ import ( "github.com/gobwas/glob" "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" goversion "github.com/hashicorp/go-version" ) @@ -37,17 +38,25 @@ const ( CTX_LATENCIES ContextType = "latencies" ) -const STORAGE_ENGINE = "storage-engine_" - // below constant represent the labels we send along with metrics to Prometheus or something -const METRIC_LABEL_CLUSTER_NAME = "cluster_name" -const METRIC_LABEL_SERVICE = "service" -const METRIC_LABEL_NS = "ns" -const METRIC_LABEL_SET = "set" -const METRIC_LABEL_LE = "le" -const METRIC_LABEL_DC_NAME = "dc" -const METRIC_LABEL_SINDEX = "sindex" -const METRIC_LABEL_USER = "user" +const ( + METRIC_LABEL_CLUSTER_NAME = "cluster_name" + METRIC_LABEL_SERVICE = "service" + METRIC_LABEL_NS = "ns" + METRIC_LABEL_SET = "set" + METRIC_LABEL_LE = "le" + METRIC_LABEL_DC_NAME = "dc" + METRIC_LABEL_INDEX = "index" + METRIC_LABEL_SINDEX = "sindex" + METRIC_LABEL_USER = "user" +) + +// constants used to identify type of metrics +const ( + STORAGE_ENGINE = "storage-engine" + INDEX_TYPE = "index-type" + SINDEX_TYPE = "sindex-type" +) func makeMetric(namespace, name string, t metricType, constLabels map[string]string, labels ...string) promMetric { promDesc := prometheus.NewDesc( @@ -424,3 +433,21 @@ func getMetricType(pContext ContextType, pRawMetricName string) metricType { return mtCounter } + +// This is a common utility, used by all the watchers to push metric to prometheus +func pushToPrometheus(asMetric AerospikeStat, pv float64, labels []string, labelValues []string, + ch chan<- prometheus.Metric) { + + if asMetric.isAllowed { + // handle any panic from prometheus, this may occur when prom encounters a config/stat with special characters + defer func() { + if r := recover(); r != nil { + log.Tracef("%s recovered from panic while handling stat %s", string(asMetric.context), asMetric.name) + } + }() + + desc, valueType := asMetric.makePromMetric(labels...) + ch <- prometheus.MustNewConstMetric(desc, valueType, pv, labelValues...) + + } +} diff --git a/gauge_stats.go b/gauge_stats.go index 642e69c8..f4945649 100644 --- a/gauge_stats.go +++ b/gauge_stats.go @@ -34,7 +34,6 @@ func initGaugeStats(pGaugeStatsFile string, pGaugeStats *GaugeStats) { log.Infof("Loading Gauge Stats file %s", pGaugeStatsFile) - // fmt.Println("Loading Gauge Stats file ", configFile) blob, err := os.ReadFile(pGaugeStatsFile) if err != nil { log.Fatalln(err) diff --git a/gauge_stats_list.toml b/gauge_stats_list.toml index ef146430..90d952d9 100644 --- a/gauge_stats_list.toml +++ b/gauge_stats_list.toml @@ -55,9 +55,7 @@ sindex_gauge_stats = [ # node_gauge_stats = [ - "batch_index_created_buffers", - "batch_index_huge_buffers", - "batch_index_proto_compression_ratio", + "batch_index_proto_compression_ratio", "batch_index_proto_uncompressed_pct", "batch_index_queue", "batch_index_unused_buffers", @@ -69,9 +67,6 @@ node_gauge_stats = [ "cluster_max_compatibility_id", "cluster_min_compatibility_id", "cluster_size", - "dlog_free_pct", - "dlog_logged", - "dlog_used_objects", "fabric_bulk_recv_rate", "fabric_bulk_send_rate", "fabric_connections", @@ -111,8 +106,9 @@ node_gauge_stats = [ "tree_gc_queue", "tsvc_queue", # - # TODO: Do we need the below XDR Stats - # + # 4.x XDR stats + "dlog_free_pct", + "dlog_used_objects", "xdr_active_failed_node_sessions", "xdr_active_link_down_sessions", "xdr_global_lastshiptime", @@ -124,8 +120,6 @@ node_gauge_stats = [ "xdr_read_respq_used", "xdr_read_txnq_used_pct", "xdr_read_txnq_used", - "xdr_relogged_incoming", - "xdr_relogged_outgoing", "xdr_ship_compression_avg_pct", "xdr_ship_inflight_objects", "xdr_ship_latency_avg", @@ -138,7 +132,6 @@ node_gauge_stats = [ # Namespace: below section define all Namespace stats which are treated as Gauges # namespace_gauge_stats =[ - "appeals_rx_active", "appeals_tx_active", "appeals_tx_remaining", @@ -171,11 +164,7 @@ namespace_gauge_stats =[ "memory_used_index_bytes", "memory_used_set_index_bytes", "memory_used_sindex_bytes", - "migrate_record_receives", - "migrate_record_retransmits", - "migrate_records_skipped", - "migrate_records_transmitted", - "migrate_rx_instances", + "migrate_rx_instances", "migrate_rx_partitions_active", "migrate_rx_partitions_initial", "migrate_rx_partitions_remaining", @@ -214,22 +203,16 @@ namespace_gauge_stats =[ "scan_proto_uncompressed_pct", "shadow_write_q", "stop_writes", - "storage-engine.device.defrag_q", - "storage-engine.device.defrag_reads", - "storage-engine.device.defrag_writes", "storage-engine.device.free_wblocks", "storage-engine.device.shadow_write_q", "storage-engine.device.used_bytes", "storage-engine.device.write_q", "storage-engine.file.defrag_q", - "storage-engine.file.defrag_reads", - "storage-engine.file.defrag_writes", "storage-engine.file.free_wblocks", "storage-engine.file.shadow_write_q", "storage-engine.file.used_bytes", - "storage-engine.file.write_q", - + "storage-engine.file.write_q", "tombstones", "truncate_lut", "unavailable_partitions", diff --git a/gauge_stats_test.go b/gauge_stats_test.go index 53ceef68..b1d62b80 100644 --- a/gauge_stats_test.go +++ b/gauge_stats_test.go @@ -2,22 +2,16 @@ package main import ( "fmt" - "os" "testing" "github.com/stretchr/testify/assert" ) func TestGetGaugesNotEmpty(t *testing.T) { - - // this is to force-reload the config in the NamespaceWatcher, this is a check on this param in NamespaceWatcher implementation - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing GaugeMetrics ... TestGetGaugesNotEmpty") // Initialize and validate Gauge config gaugeList := new(GaugeStats) - initGaugeStats(METRICS_CONFIG_FILE, gaugeList) nslist := gaugeList.NamespaceStats @@ -27,10 +21,6 @@ func TestGetGaugesNotEmpty(t *testing.T) { } func TestGetGaugesCounts(t *testing.T) { - - // this is to force-reload the config in the NamespaceWatcher, this is a check on this param in NamespaceWatcher implementation - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing GaugeMetrics ... TestGetGaugesCounts") // Initialize and validate Gauge config @@ -39,10 +29,10 @@ func TestGetGaugesCounts(t *testing.T) { initGaugeStats(METRICS_CONFIG_FILE, gaugeList) glist := gaugeList.NamespaceStats - assert.Equal(t, len(glist), 96) + assert.Equal(t, len(glist), 88) glist = gaugeList.NodeStats - assert.Equal(t, len(glist), 74) + assert.Equal(t, len(glist), 69) glist = gaugeList.SetsStats assert.Equal(t, len(glist), 7) @@ -56,9 +46,7 @@ func TestGetGaugesCounts(t *testing.T) { } func TestIsAGaugeTrue(t *testing.T) { - - // this is to force-reload the config in the NamespaceWatcher, this is a check on this param in NamespaceWatcher implementation - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) + fmt.Println("initializing GaugeMetrics ... TestIsAGaugeTrue") // Initialize and validate Gauge config gaugeList := new(GaugeStats) @@ -86,9 +74,6 @@ func TestIsAGaugeTrue(t *testing.T) { func TestNoGaugeExists(t *testing.T) { - // this is to force-reload the config in the NamespaceWatcher, this is a check on this param in NamespaceWatcher implementation - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing GaugeMetrics ... TestNoGaugeExists") // Initialize and validate Gauge config diff --git a/helper_test.go b/helper_test.go index e9db0fbf..fd42e254 100644 --- a/helper_test.go +++ b/helper_test.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "sort" "strconv" "strings" @@ -13,13 +14,9 @@ var LABELS_APE_TOML = "tests/labels_ape.toml" var NS_ALLOWLIST_APE_TOML = "tests/ns_allowlist_ape.toml" var NS_BLOCKLIST_APE_TOML = "tests/ns_blocklist_ape.toml" -var TESTCASE_MODE = "TESTCASE_MODE" -var TESTCASE_MODE_TRUE = "true" -var TESTCASE_MODE_FALSE = "false" - var METRICS_CONFIG_FILE = "gauge_stats_list.toml" -// var g_ns_metric_allow_list = []string{"aerospike_namespace_master_objects", "aerospike_namespace_memory_used_bytes"} +var MOCK_TEST_DATA_FILE = "tests/mock_test_data.txt" func extractNamespaceFromLabel(label string) string { // [name:"cluster_name" value:"" name:"ns" value:"bar" name:"service" value:"" ] @@ -30,6 +27,14 @@ func extractNamespaceFromLabel(label string) string { return nsFromLabel } +func stringifyLabel(label string) string { + labelReplacerFunc := strings.NewReplacer(".", "_", "-", "_", " ", "_", "[", "_", "]", "_", "\"", "_", ":", "_", "/", "_") + hypenReplacerFunc := strings.NewReplacer("_", "") + + // return labelReplacerFunc.Replace(label) + return hypenReplacerFunc.Replace(labelReplacerFunc.Replace(label)) +} + func extractLabelNameValueFromFullLabel(fullLabel string, reqName string) string { // Example Given Original: [name:"cluster_name" value:"null" name:"service" value:"172.17.0.3:3000" ] @@ -256,3 +261,35 @@ func extractOperationFromMetric(metricName string) string { return operationName } + +func copyConfigLabels() map[string]string { + cfgLabels := config.AeroProm.MetricLabels + + returnLabelMap := make(map[string]string) + for key, value := range cfgLabels { + returnLabelMap[key] = value + } + + return returnLabelMap +} + +func createLabelByNames(labelsMap map[string]string) string { + + arr_label_names := []string{} + createdLabelString := "" + + for key := range labelsMap { + arr_label_names = append(arr_label_names, strings.TrimSpace(key)) + } + + // sort the keys + sort.Strings(arr_label_names) + + for idx := range arr_label_names { + keyName := arr_label_names[idx] + value := labelsMap[keyName] + createdLabelString = createdLabelString + constructLabelElement(keyName, strings.TrimSpace(value)) + } + + return strings.TrimSpace(createdLabelString) +} diff --git a/mock_data_generator_test.go b/mock_data_generator_test.go index 7f7912b1..4fcbf353 100644 --- a/mock_data_generator_test.go +++ b/mock_data_generator_test.go @@ -1,9 +1,10 @@ package main import ( + "bufio" "fmt" + "os" "regexp" - "sort" "strconv" "strings" @@ -19,102 +20,276 @@ once we have output from watcher-implementations ( like watcher_namespaces.go, w this output is compated with the expected results generated by Test-Cases */ -func getRawMetrics() map[string]string { +type MockAerospikeServer struct { + namespaces_stats []string + sets_stats []string + xdr_stats []string + node_stats []string + latencies_stats []string + sindex_stats []string + build []string + cluster_name []string + service_clear_std []string + passone_output_str string + passone_outputs_map map[string]string +} + +// read mock test data from a file +func (md *MockAerospikeServer) initialize() { + + filePath := MOCK_TEST_DATA_FILE + readFile, err := os.Open(filePath) + + if err != nil { + fmt.Println(err) + } + fileScanner := bufio.NewScanner(readFile) + fileScanner.Split(bufio.ScanLines) + var fileLines []string + + for fileScanner.Scan() { + fileLines = append(fileLines, fileScanner.Text()) + } + + readFile.Close() + + for _, line := range fileLines { + if strings.HasPrefix(line, "#") && strings.HasPrefix(line, "//") { + // ignore, comments + } else if len(line) > 0 { + if strings.HasPrefix(line, "namespace-") { + md.namespaces_stats = append(md.namespaces_stats, line) + } else if strings.HasPrefix(line, "set-stats") { + md.sets_stats = append(md.sets_stats, line) + } else if strings.HasPrefix(line, "latency-stats") { + md.latencies_stats = append(md.latencies_stats, line) + } else if strings.HasPrefix(line, "node-") { + md.node_stats = append(md.node_stats, line) + } else if strings.HasPrefix(line, "xdr-") { + md.xdr_stats = append(md.xdr_stats, line) + } else if strings.HasPrefix(line, "sindex-") { + md.sindex_stats = append(md.sindex_stats, line) + } else if strings.HasPrefix(line, "build") { + md.build = append(md.build, line) + } else if strings.HasPrefix(line, "service-clear-std") { + md.service_clear_std = append(md.service_clear_std, line) + } else if strings.HasPrefix(line, "cluster-name") { + md.cluster_name = append(md.cluster_name, line) + } else if strings.HasPrefix(line, "passone_output") { + // passone_output:build:6.4.0.0-rc4 get-config:context=xdr:dcs=backup_dc_asdev20,backup_dc_asdev20_second;src-id=0;trace-sample=0 namespaces:test;bar_device;materials;ns_test_on_flash;test_on_shmem;bar_on_flash;pmkohl_on_device sindex:ns=test:indexname=test_sindex1:set=from_branch_2:bin=occurred:type=numeric:indextype=default:context=null:state=RW + str := strings.ReplaceAll(line, "passone_output:", "") + // store full string also + md.passone_output_str = str + elements := strings.Split(md.passone_output_str, " ") + // reinitialize internal map + md.passone_outputs_map = make(map[string]string) + for _, entry := range elements { + + colonIndex := strings.Index(entry, ":") + // parts := strings.Split(entry, ":") + key := entry[0:colonIndex] + value := entry[colonIndex+1:] + md.passone_outputs_map[key] = value + } + // md.passone_outputs = splitAndRetrieveStats(str, ";") + + } + + } + } +} + +func (md *MockAerospikeServer) fetchRawMetrics() map[string]string { rawMetrics := make(map[string]string) - // ORIGINAL TEST DATA == DONT DELETE - rawMetrics["namespace/test"] = "ns_cluster_size=1;effective_replication_factor=1;objects=38914;tombstones=0;xdr_tombstones=0;xdr_bin_cemeteries=0;master_objects=38914;master_tombstones=0;prole_objects=0;prole_tombstones=0;non_replica_objects=0;non_replica_tombstones=0;unreplicated_records=0;dead_partitions=0;unavailable_partitions=0;clock_skew_stop_writes=false;stop_writes=false;hwm_breached=false;current_time=420112506;non_expirable_objects=0;expired_objects=0;evicted_objects=0;evict_ttl=0;evict_void_time=0;smd_evict_void_time=0;nsup_cycle_duration=0;nsup_cycle_deleted_pct=0.00;truncate_lut=0;truncating=false;sindex_gc_cleaned=0;memory_used_bytes=21830135;memory_used_data_bytes=2562423;memory_used_index_bytes=2490496;memory_used_set_index_bytes=0;memory_used_sindex_bytes=16777216;memory_free_pct=99;xmem_id=1;available_bin_names=65529;record_proto_uncompressed_pct=0.000;record_proto_compression_ratio=1.000;query_proto_uncompressed_pct=0.000;query_proto_compression_ratio=1.000;pending_quiesce=false;effective_is_quiesced=false;nodes_quiesced=0;effective_prefer_uniform_balance=true;migrate_tx_partitions_imbalance=0;migrate_tx_instances=0;migrate_rx_instances=0;migrate_tx_partitions_active=0;migrate_rx_partitions_active=0;migrate_tx_partitions_initial=0;migrate_tx_partitions_remaining=0;migrate_tx_partitions_lead_remaining=0;migrate_rx_partitions_initial=0;migrate_rx_partitions_remaining=0;migrate_records_skipped=0;migrate_records_transmitted=0;migrate_record_retransmits=0;migrate_record_receives=0;migrate_signals_active=0;migrate_signals_remaining=0;appeals_tx_active=0;appeals_rx_active=0;appeals_tx_remaining=0;appeals_records_exonerated=0;client_tsvc_error=0;client_tsvc_timeout=0;client_proxy_complete=0;client_proxy_error=0;client_proxy_timeout=0;client_read_success=126;client_read_error=0;client_read_timeout=0;client_read_not_found=27;client_read_filtered_out=0;client_write_success=64414;client_write_error=0;client_write_timeout=0;client_write_filtered_out=0;xdr_client_write_success=0;xdr_client_write_error=0;xdr_client_write_timeout=0;client_delete_success=20;client_delete_error=0;client_delete_timeout=0;client_delete_not_found=0;client_delete_filtered_out=0;xdr_client_delete_success=0;xdr_client_delete_error=0;xdr_client_delete_timeout=0;xdr_client_delete_not_found=0;client_udf_complete=0;client_udf_error=0;client_udf_timeout=0;client_udf_filtered_out=0;client_lang_read_success=0;client_lang_write_success=0;client_lang_delete_success=0;client_lang_error=0;from_proxy_tsvc_error=0;from_proxy_tsvc_timeout=0;from_proxy_read_success=0;from_proxy_read_error=0;from_proxy_read_timeout=0;from_proxy_read_not_found=0;from_proxy_read_filtered_out=0;from_proxy_write_success=0;from_proxy_write_error=0;from_proxy_write_timeout=0;from_proxy_write_filtered_out=0;xdr_from_proxy_write_success=0;xdr_from_proxy_write_error=0;xdr_from_proxy_write_timeout=0;from_proxy_delete_success=0;from_proxy_delete_error=0;from_proxy_delete_timeout=0;from_proxy_delete_not_found=0;from_proxy_delete_filtered_out=0;xdr_from_proxy_delete_success=0;xdr_from_proxy_delete_error=0;xdr_from_proxy_delete_timeout=0;xdr_from_proxy_delete_not_found=0;from_proxy_udf_complete=0;from_proxy_udf_error=0;from_proxy_udf_timeout=0;from_proxy_udf_filtered_out=0;from_proxy_lang_read_success=0;from_proxy_lang_write_success=0;from_proxy_lang_delete_success=0;from_proxy_lang_error=0;batch_sub_tsvc_error=0;batch_sub_tsvc_timeout=0;batch_sub_proxy_complete=0;batch_sub_proxy_error=0;batch_sub_proxy_timeout=0;batch_sub_read_success=0;batch_sub_read_error=0;batch_sub_read_timeout=0;batch_sub_read_not_found=0;batch_sub_read_filtered_out=0;batch_sub_write_success=0;batch_sub_write_error=0;batch_sub_write_timeout=0;batch_sub_write_filtered_out=0;batch_sub_delete_success=0;batch_sub_delete_error=0;batch_sub_delete_timeout=0;batch_sub_delete_not_found=0;batch_sub_delete_filtered_out=0;batch_sub_udf_complete=0;batch_sub_udf_error=0;batch_sub_udf_timeout=0;batch_sub_udf_filtered_out=0;batch_sub_lang_read_success=0;batch_sub_lang_write_success=0;batch_sub_lang_delete_success=0;batch_sub_lang_error=0;from_proxy_batch_sub_tsvc_error=0;from_proxy_batch_sub_tsvc_timeout=0;from_proxy_batch_sub_read_success=0;from_proxy_batch_sub_read_error=0;from_proxy_batch_sub_read_timeout=0;from_proxy_batch_sub_read_not_found=0;from_proxy_batch_sub_read_filtered_out=0;from_proxy_batch_sub_write_success=0;from_proxy_batch_sub_write_error=0;from_proxy_batch_sub_write_timeout=0;from_proxy_batch_sub_write_filtered_out=0;from_proxy_batch_sub_delete_success=0;from_proxy_batch_sub_delete_error=0;from_proxy_batch_sub_delete_timeout=0;from_proxy_batch_sub_delete_not_found=0;from_proxy_batch_sub_delete_filtered_out=0;from_proxy_batch_sub_udf_complete=0;from_proxy_batch_sub_udf_error=0;from_proxy_batch_sub_udf_timeout=0;from_proxy_batch_sub_udf_filtered_out=0;from_proxy_batch_sub_lang_read_success=0;from_proxy_batch_sub_lang_write_success=0;from_proxy_batch_sub_lang_delete_success=0;from_proxy_batch_sub_lang_error=0;udf_sub_tsvc_error=0;udf_sub_tsvc_timeout=0;udf_sub_udf_complete=0;udf_sub_udf_error=0;udf_sub_udf_timeout=0;udf_sub_udf_filtered_out=0;udf_sub_lang_read_success=0;udf_sub_lang_write_success=0;udf_sub_lang_delete_success=0;udf_sub_lang_error=0;ops_sub_tsvc_error=0;ops_sub_tsvc_timeout=0;ops_sub_write_success=0;ops_sub_write_error=0;ops_sub_write_timeout=0;ops_sub_write_filtered_out=0;dup_res_ask=0;dup_res_respond_read=0;dup_res_respond_no_read=0;retransmit_all_read_dup_res=0;retransmit_all_write_dup_res=0;retransmit_all_write_repl_write=0;retransmit_all_delete_dup_res=0;retransmit_all_delete_repl_write=0;retransmit_all_udf_dup_res=0;retransmit_all_udf_repl_write=0;retransmit_all_batch_sub_dup_res=0;retransmit_udf_sub_dup_res=0;retransmit_udf_sub_repl_write=0;retransmit_ops_sub_dup_res=0;retransmit_ops_sub_repl_write=0;pi_query_short_basic_complete=0;pi_query_short_basic_error=0;pi_query_short_basic_timeout=0;pi_query_long_basic_complete=1;pi_query_long_basic_error=14;pi_query_long_basic_abort=0;pi_query_aggr_complete=0;pi_query_aggr_error=0;pi_query_aggr_abort=0;pi_query_udf_bg_complete=0;pi_query_udf_bg_error=0;pi_query_udf_bg_abort=0;pi_query_ops_bg_complete=0;pi_query_ops_bg_error=0;pi_query_ops_bg_abort=0;si_query_short_basic_complete=0;si_query_short_basic_error=0;si_query_short_basic_timeout=0;si_query_long_basic_complete=0;si_query_long_basic_error=0;si_query_long_basic_abort=0;si_query_aggr_complete=0;si_query_aggr_error=0;si_query_aggr_abort=0;si_query_udf_bg_complete=0;si_query_udf_bg_error=0;si_query_udf_bg_abort=0;si_query_ops_bg_complete=0;si_query_ops_bg_error=0;si_query_ops_bg_abort=0;geo_region_query_reqs=0;geo_region_query_cells=0;geo_region_query_points=0;geo_region_query_falsepos=0;re_repl_tsvc_error=0;re_repl_tsvc_timeout=0;re_repl_success=0;re_repl_error=0;re_repl_timeout=0;fail_xdr_forbidden=0;fail_key_busy=0;fail_generation=0;fail_record_too_big=0;fail_client_lost_conflict=0;fail_xdr_lost_conflict=0;deleted_last_bin=0;allow-ttl-without-nsup=false;background-query-max-rps=10000;conflict-resolution-policy=generation;conflict-resolve-writes=false;data-in-index=false;default-ttl=0;disable-cold-start-eviction=false;disable-write-dup-res=false;disallow-expunge=false;disallow-null-setname=false;enable-benchmarks-batch-sub=false;enable-benchmarks-ops-sub=false;enable-benchmarks-read=false;enable-benchmarks-udf=false;enable-benchmarks-udf-sub=false;enable-benchmarks-write=false;enable-hist-proxy=false;evict-hist-buckets=10000;evict-tenths-pct=5;force-long-queries=false;high-water-disk-pct=0;high-water-memory-pct=0;ignore-migrate-fill-delay=false;index-stage-size=1073741824;inline-short-queries=false;max-record-size=0;memory-size=4294967296;migrate-order=5;migrate-retransmit-ms=5000;migrate-sleep=1;nsup-hist-period=3600;nsup-period=0;nsup-threads=1;partition-tree-sprigs=256;prefer-uniform-balance=true;rack-id=0;read-consistency-level-override=off;reject-non-xdr-writes=false;reject-xdr-writes=false;replication-factor=2;sindex-stage-size=1073741824;single-bin=false;single-query-threads=4;stop-writes-pct=90;stop-writes-sys-memory-pct=90;strong-consistency=false;strong-consistency-allow-expunge=false;tomb-raider-eligible-age=86400;tomb-raider-period=86400;transaction-pending-limit=20;truncate-threads=4;write-commit-level-override=off;xdr-bin-tombstone-ttl=86400;xdr-tomb-raider-period=120;xdr-tomb-raider-threads=1;geo2dsphere-within.strict=true;geo2dsphere-within.min-level=1;geo2dsphere-within.max-level=20;geo2dsphere-within.max-cells=12;geo2dsphere-within.level-mod=1;geo2dsphere-within.earth-radius-meters=6371000;index-type=shmem;sindex-type=shmem;storage-engine=memory" - rawMetrics["namespace/bar"] = "ns_cluster_size=1;effective_replication_factor=1;objects=13403218;tombstones=0;xdr_tombstones=0;xdr_bin_cemeteries=0;master_objects=13403218;master_tombstones=0;prole_objects=0;prole_tombstones=0;non_replica_objects=0;non_replica_tombstones=0;unreplicated_records=0;dead_partitions=0;unavailable_partitions=0;clock_skew_stop_writes=false;stop_writes=false;hwm_breached=false;current_time=420112506;non_expirable_objects=0;expired_objects=0;evicted_objects=0;evict_ttl=0;evict_void_time=0;smd_evict_void_time=0;nsup_cycle_duration=0;nsup_cycle_deleted_pct=0.00;truncate_lut=0;truncating=false;sindex_gc_cleaned=0;memory_used_bytes=1751784700;memory_used_data_bytes=893978748;memory_used_index_bytes=857805952;memory_used_set_index_bytes=0;memory_used_sindex_bytes=0;memory_free_pct=59;xmem_id=2;available_bin_names=65532;device_total_bytes=17179869184;device_used_bytes=1621031664;device_free_pct=90;device_available_pct=86;storage-engine.file[0].used_bytes=1621031664;storage-engine.file[0].free_wblocks=14198;storage-engine.file[0].write_q=0;storage-engine.file[0].writes=0;storage-engine.file[0].defrag_q=0;storage-engine.file[0].defrag_reads=2;storage-engine.file[0].defrag_writes=0;storage-engine.file[0].shadow_write_q=0;storage-engine.file[0].age=-1;record_proto_uncompressed_pct=0.000;record_proto_compression_ratio=1.000;query_proto_uncompressed_pct=0.000;query_proto_compression_ratio=1.000;pending_quiesce=false;effective_is_quiesced=false;nodes_quiesced=0;effective_prefer_uniform_balance=true;migrate_tx_partitions_imbalance=0;migrate_tx_instances=0;migrate_rx_instances=0;migrate_tx_partitions_active=0;migrate_rx_partitions_active=0;migrate_tx_partitions_initial=0;migrate_tx_partitions_remaining=0;migrate_tx_partitions_lead_remaining=0;migrate_rx_partitions_initial=0;migrate_rx_partitions_remaining=0;migrate_records_skipped=0;migrate_records_transmitted=0;migrate_record_retransmits=0;migrate_record_receives=0;migrate_signals_active=0;migrate_signals_remaining=0;appeals_tx_active=0;appeals_rx_active=0;appeals_tx_remaining=0;appeals_records_exonerated=0;client_tsvc_error=0;client_tsvc_timeout=0;client_proxy_complete=0;client_proxy_error=0;client_proxy_timeout=0;client_read_success=0;client_read_error=0;client_read_timeout=0;client_read_not_found=0;client_read_filtered_out=0;client_write_success=13;client_write_error=0;client_write_timeout=0;client_write_filtered_out=0;xdr_client_write_success=0;xdr_client_write_error=0;xdr_client_write_timeout=0;client_delete_success=0;client_delete_error=0;client_delete_timeout=0;client_delete_not_found=0;client_delete_filtered_out=0;xdr_client_delete_success=0;xdr_client_delete_error=0;xdr_client_delete_timeout=0;xdr_client_delete_not_found=0;client_udf_complete=0;client_udf_error=0;client_udf_timeout=0;client_udf_filtered_out=0;client_lang_read_success=0;client_lang_write_success=0;client_lang_delete_success=0;client_lang_error=0;from_proxy_tsvc_error=0;from_proxy_tsvc_timeout=0;from_proxy_read_success=0;from_proxy_read_error=0;from_proxy_read_timeout=0;from_proxy_read_not_found=0;from_proxy_read_filtered_out=0;from_proxy_write_success=0;from_proxy_write_error=0;from_proxy_write_timeout=0;from_proxy_write_filtered_out=0;xdr_from_proxy_write_success=0;xdr_from_proxy_write_error=0;xdr_from_proxy_write_timeout=0;from_proxy_delete_success=0;from_proxy_delete_error=0;from_proxy_delete_timeout=0;from_proxy_delete_not_found=0;from_proxy_delete_filtered_out=0;xdr_from_proxy_delete_success=0;xdr_from_proxy_delete_error=0;xdr_from_proxy_delete_timeout=0;xdr_from_proxy_delete_not_found=0;from_proxy_udf_complete=0;from_proxy_udf_error=0;from_proxy_udf_timeout=0;from_proxy_udf_filtered_out=0;from_proxy_lang_read_success=0;from_proxy_lang_write_success=0;from_proxy_lang_delete_success=0;from_proxy_lang_error=0;batch_sub_tsvc_error=0;batch_sub_tsvc_timeout=0;batch_sub_proxy_complete=0;batch_sub_proxy_error=0;batch_sub_proxy_timeout=0;batch_sub_read_success=0;batch_sub_read_error=0;batch_sub_read_timeout=0;batch_sub_read_not_found=0;batch_sub_read_filtered_out=0;batch_sub_write_success=0;batch_sub_write_error=0;batch_sub_write_timeout=0;batch_sub_write_filtered_out=0;batch_sub_delete_success=0;batch_sub_delete_error=0;batch_sub_delete_timeout=0;batch_sub_delete_not_found=0;batch_sub_delete_filtered_out=0;batch_sub_udf_complete=0;batch_sub_udf_error=0;batch_sub_udf_timeout=0;batch_sub_udf_filtered_out=0;batch_sub_lang_read_success=0;batch_sub_lang_write_success=0;batch_sub_lang_delete_success=0;batch_sub_lang_error=0;from_proxy_batch_sub_tsvc_error=0;from_proxy_batch_sub_tsvc_timeout=0;from_proxy_batch_sub_read_success=0;from_proxy_batch_sub_read_error=0;from_proxy_batch_sub_read_timeout=0;from_proxy_batch_sub_read_not_found=0;from_proxy_batch_sub_read_filtered_out=0;from_proxy_batch_sub_write_success=0;from_proxy_batch_sub_write_error=0;from_proxy_batch_sub_write_timeout=0;from_proxy_batch_sub_write_filtered_out=0;from_proxy_batch_sub_delete_success=0;from_proxy_batch_sub_delete_error=0;from_proxy_batch_sub_delete_timeout=0;from_proxy_batch_sub_delete_not_found=0;from_proxy_batch_sub_delete_filtered_out=0;from_proxy_batch_sub_udf_complete=0;from_proxy_batch_sub_udf_error=0;from_proxy_batch_sub_udf_timeout=0;from_proxy_batch_sub_udf_filtered_out=0;from_proxy_batch_sub_lang_read_success=0;from_proxy_batch_sub_lang_write_success=0;from_proxy_batch_sub_lang_delete_success=0;from_proxy_batch_sub_lang_error=0;udf_sub_tsvc_error=0;udf_sub_tsvc_timeout=0;udf_sub_udf_complete=0;udf_sub_udf_error=0;udf_sub_udf_timeout=0;udf_sub_udf_filtered_out=0;udf_sub_lang_read_success=0;udf_sub_lang_write_success=0;udf_sub_lang_delete_success=0;udf_sub_lang_error=0;ops_sub_tsvc_error=0;ops_sub_tsvc_timeout=0;ops_sub_write_success=0;ops_sub_write_error=0;ops_sub_write_timeout=0;ops_sub_write_filtered_out=0;dup_res_ask=0;dup_res_respond_read=0;dup_res_respond_no_read=0;retransmit_all_read_dup_res=0;retransmit_all_write_dup_res=0;retransmit_all_write_repl_write=0;retransmit_all_delete_dup_res=0;retransmit_all_delete_repl_write=0;retransmit_all_udf_dup_res=0;retransmit_all_udf_repl_write=0;retransmit_all_batch_sub_dup_res=0;retransmit_udf_sub_dup_res=0;retransmit_udf_sub_repl_write=0;retransmit_ops_sub_dup_res=0;retransmit_ops_sub_repl_write=0;pi_query_short_basic_complete=0;pi_query_short_basic_error=0;pi_query_short_basic_timeout=0;pi_query_long_basic_complete=0;pi_query_long_basic_error=0;pi_query_long_basic_abort=0;pi_query_aggr_complete=0;pi_query_aggr_error=0;pi_query_aggr_abort=0;pi_query_udf_bg_complete=0;pi_query_udf_bg_error=0;pi_query_udf_bg_abort=0;pi_query_ops_bg_complete=0;pi_query_ops_bg_error=0;pi_query_ops_bg_abort=0;si_query_short_basic_complete=0;si_query_short_basic_error=0;si_query_short_basic_timeout=0;si_query_long_basic_complete=0;si_query_long_basic_error=0;si_query_long_basic_abort=0;si_query_aggr_complete=0;si_query_aggr_error=0;si_query_aggr_abort=0;si_query_udf_bg_complete=0;si_query_udf_bg_error=0;si_query_udf_bg_abort=0;si_query_ops_bg_complete=0;si_query_ops_bg_error=0;si_query_ops_bg_abort=0;geo_region_query_reqs=0;geo_region_query_cells=0;geo_region_query_points=0;geo_region_query_falsepos=0;re_repl_tsvc_error=0;re_repl_tsvc_timeout=0;re_repl_success=0;re_repl_error=0;re_repl_timeout=0;fail_xdr_forbidden=0;fail_key_busy=0;fail_generation=0;fail_record_too_big=0;fail_client_lost_conflict=0;fail_xdr_lost_conflict=0;deleted_last_bin=0;allow-ttl-without-nsup=false;background-query-max-rps=10000;conflict-resolution-policy=generation;conflict-resolve-writes=false;data-in-index=false;default-ttl=0;disable-cold-start-eviction=false;disable-write-dup-res=false;disallow-expunge=false;disallow-null-setname=false;enable-benchmarks-batch-sub=false;enable-benchmarks-ops-sub=false;enable-benchmarks-read=false;enable-benchmarks-udf=false;enable-benchmarks-udf-sub=false;enable-benchmarks-write=false;enable-hist-proxy=false;evict-hist-buckets=10000;evict-tenths-pct=5;force-long-queries=false;high-water-disk-pct=0;high-water-memory-pct=0;ignore-migrate-fill-delay=false;index-stage-size=1073741824;inline-short-queries=false;max-record-size=0;memory-size=4294967296;migrate-order=5;migrate-retransmit-ms=5000;migrate-sleep=1;nsup-hist-period=3600;nsup-period=0;nsup-threads=1;partition-tree-sprigs=256;prefer-uniform-balance=true;rack-id=0;read-consistency-level-override=off;reject-non-xdr-writes=false;reject-xdr-writes=false;replication-factor=2;sindex-stage-size=1073741824;single-bin=false;single-query-threads=4;stop-writes-pct=90;stop-writes-sys-memory-pct=90;strong-consistency=false;strong-consistency-allow-expunge=false;tomb-raider-eligible-age=86400;tomb-raider-period=86400;transaction-pending-limit=20;truncate-threads=4;write-commit-level-override=off;xdr-bin-tombstone-ttl=86400;xdr-tomb-raider-period=120;xdr-tomb-raider-threads=1;geo2dsphere-within.strict=true;geo2dsphere-within.min-level=1;geo2dsphere-within.max-level=20;geo2dsphere-within.max-cells=12;geo2dsphere-within.level-mod=1;geo2dsphere-within.earth-radius-meters=6371000;index-type=shmem;sindex-type=shmem;storage-engine=device;storage-engine.file[0]=/opt/aerospike/data/bar.dat;storage-engine.cache-replica-writes=false;storage-engine.cold-start-empty=false;storage-engine.commit-to-device=false;storage-engine.commit-min-size=0;storage-engine.compression=none;storage-engine.compression-acceleration=0;storage-engine.compression-level=0;storage-engine.data-in-memory=true;storage-engine.defrag-lwm-pct=50;storage-engine.defrag-queue-min=0;storage-engine.defrag-sleep=1000;storage-engine.defrag-startup-minimum=0;storage-engine.direct-files=false;storage-engine.disable-odsync=false;storage-engine.enable-benchmarks-storage=false;storage-engine.encryption-key-file=null;storage-engine.encryption-old-key-file=null;storage-engine.filesize=17179869184;storage-engine.flush-max-ms=1000;storage-engine.max-used-pct=70;storage-engine.max-write-cache=67108864;storage-engine.min-avail-pct=5;storage-engine.post-write-queue=0;storage-engine.read-page-cache=false;storage-engine.scheduler-mode=null;storage-engine.serialize-tomb-raider=false;storage-engine.sindex-startup-device-scan=false;storage-engine.tomb-raider-sleep=1000;storage-engine.write-block-size=1048576" - rawMetrics["sets"] = "ns=test:set=test_set2:objects=12973:tombstones=0:memory_data_bytes=845591:device_data_bytes=0:truncate_lut=0:sindexes=1:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=test:set=test_east_region:objects=12971:tombstones=0:memory_data_bytes=858449:device_data_bytes=0:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=test:set=test_vendors:objects=12970:tombstones=0:memory_data_bytes=858383:device_data_bytes=0:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=test:set=aerospike:objects=0:tombstones=0:memory_data_bytes=0:device_data_bytes=0:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=test:set=ownset:objects=0:tombstones=0:memory_data_bytes=0:device_data_bytes=0:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=bar:set=set2:objects=3684785:tombstones=0:memory_data_bytes=247974882:device_data_bytes=412695920:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=bar:set=vendors:objects=3495040:tombstones=0:memory_data_bytes=234530114:device_data_bytes=411741440:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=bar:set=south_region:objects=998985:tombstones=0:memory_data_bytes=65825013:device_data_bytes=127870080:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=bar:set=north_region:objects=998987:tombstones=0:memory_data_bytes=65825146:device_data_bytes=127870336:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=bar:set=east_region:objects=3226447:tombstones=0:memory_data_bytes=213999303:device_data_bytes=412985216:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0;ns=bar:set=west_region:objects=998974:tombstones=0:memory_data_bytes=65824290:device_data_bytes=127868672:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0" - rawMetrics["latencies:"] = "batch-index:;{test}-read:msec,0.0,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00;{test}-write:msec,0.0,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00;{test}-udf:;{test}-pi-query:;{test}-si-query:;{bar}-read:;{bar}-write:msec,0.0,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00;{bar}-udf:;{bar}-pi-query:;{bar}-si-query:" - rawMetrics["sindex/test/test_sindex1"] = "entries=12972;used_bytes=16777216;entries_per_bval=12972;entries_per_rec=1;load_pct=100;load_time=0;stat_gc_recs=0" - rawMetrics["query-show"] = "trid=12860712447849720134:ns=test:set=test_set2:n-pids-requested=4096:n-keyds-requested=0:rps=0:active-threads=0:status=done(ok):job-progress=100.00:run-time=91:time-since-done=76964462:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=12971:recs-failed=0:from=172.17.0.6+40714:job-type=basic:net-io-bytes=2116680:net-io-time=45:socket-timeout=30000;trid=10212667924988677130:ns=test:set=test_vendors:n-pids-requested=4090:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96971708:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2192:recs-failed=0:from=172.17.0.2+53556:job-type=basic:net-io-bytes=232535:net-io-time=0:socket-timeout=30000;trid=9038054471030658800:ns=test:set=test_vendors:n-pids-requested=4091:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96971747:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2193:recs-failed=0:from=172.17.0.2+53548:job-type=basic:net-io-bytes=228561:net-io-time=0:socket-timeout=30000;trid=18297828132092499463:ns=test:set=test_vendors:n-pids-requested=4092:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96971786:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2193:recs-failed=0:from=172.17.0.2+53532:job-type=basic:net-io-bytes=226885:net-io-time=0:socket-timeout=30000;trid=16496035235509212077:ns=test:set=test_vendors:n-pids-requested=4093:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96971825:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2194:recs-failed=0:from=172.17.0.2+53518:job-type=basic:net-io-bytes=228308:net-io-time=0:socket-timeout=30000;trid=5499606219233706254:ns=test:set=test_vendors:n-pids-requested=4094:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96971864:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2194:recs-failed=0:from=172.17.0.2+53510:job-type=basic:net-io-bytes=231002:net-io-time=0:socket-timeout=30000;trid=4355246641643620061:ns=test:set=test_vendors:n-pids-requested=4095:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96971904:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2196:recs-failed=0:from=172.17.0.2+53502:job-type=basic:net-io-bytes=229269:net-io-time=0:socket-timeout=30000;trid=10994964236021603577:ns=test:set=test_vendors:n-pids-requested=4096:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96971943:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2197:recs-failed=0:from=172.17.0.2+53500:job-type=basic:net-io-bytes=231177:net-io-time=0:socket-timeout=30000;trid=13990051320060701253:ns=test:set=test_vendors:n-pids-requested=4090:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96983686:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2192:recs-failed=0:from=172.17.0.2+35732:job-type=basic:net-io-bytes=218025:net-io-time=0:socket-timeout=30000;trid=13506735252273329484:ns=test:set=test_vendors:n-pids-requested=4091:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=37:time-since-done=96983725:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2192:recs-failed=0:from=172.17.0.2+35720:job-type=basic:net-io-bytes=226786:net-io-time=0:socket-timeout=30000;trid=7603703035483518258:ns=test:set=test_vendors:n-pids-requested=4092:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96983766:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2192:recs-failed=0:from=172.17.0.2+35710:job-type=basic:net-io-bytes=228998:net-io-time=0:socket-timeout=30000;trid=13591277360630725735:ns=test:set=test_vendors:n-pids-requested=4093:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96983805:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2193:recs-failed=0:from=172.17.0.2+35702:job-type=basic:net-io-bytes=227783:net-io-time=0:socket-timeout=30000;trid=8120506070021718461:ns=test:set=test_vendors:n-pids-requested=4094:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96983845:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2194:recs-failed=0:from=172.17.0.2+35690:job-type=basic:net-io-bytes=340535:net-io-time=0:socket-timeout=30000;trid=1896955730142066978:ns=test:set=test_vendors:n-pids-requested=4095:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=36:time-since-done=96983884:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2196:recs-failed=0:from=172.17.0.2+35676:job-type=basic:net-io-bytes=341889:net-io-time=0:socket-timeout=30000;trid=2711297642618297495:ns=test:set=test_vendors:n-pids-requested=4096:n-keyds-requested=0:rps=0:active-threads=0:status=done(abandoned-response-error):job-progress=100.00:run-time=43:time-since-done=96983922:recs-throttled=0:recs-filtered-meta=0:recs-filtered-bins=0:recs-succeeded=2197:recs-failed=0:from=172.17.0.2+35674:job-type=basic:net-io-bytes=226042:net-io-time=0:socket-timeout=30000" - rawMetrics["statistics"] = "failed_best_practices=true;cluster_size=1;cluster_key=4B8274A4C334;cluster_generation=1;cluster_principal=BB9030011AC4202;cluster_min_compatibility_id=11;cluster_max_compatibility_id=11;cluster_integrity=true;cluster_is_member=true;cluster_duplicate_nodes=null;cluster_clock_skew_stop_writes_sec=0;cluster_clock_skew_ms=0;cluster_clock_skew_outliers=null;uptime=97961;system_total_cpu_pct=6;system_user_cpu_pct=4;system_kernel_cpu_pct=2;system_free_mem_kbytes=9100968;system_free_mem_pct=74;system_thp_mem_kbytes=59392;process_cpu_pct=4;threads_joinable=9;threads_detached=70;threads_pool_total=31;threads_pool_active=27;heap_allocated_kbytes=1313306;heap_active_kbytes=1314744;heap_mapped_kbytes=1390080;heap_efficiency_pct=100;heap_site_count=0;objects=13442132;tombstones=0;info_queue=0;rw_in_progress=0;proxy_in_progress=0;tree_gc_queue=0;client_connections=2;client_connections_opened=5190;client_connections_closed=5188;heartbeat_connections=0;heartbeat_connections_opened=0;heartbeat_connections_closed=0;fabric_connections=0;fabric_connections_opened=0;fabric_connections_closed=0;heartbeat_received_self=0;heartbeat_received_foreign=0;reaped_fds=0;info_complete=108816;info_timeout=0;demarshal_error=0;early_tsvc_client_error=0;early_tsvc_from_proxy_error=0;early_tsvc_batch_sub_error=0;early_tsvc_from_proxy_batch_sub_error=0;early_tsvc_udf_sub_error=0;early_tsvc_ops_sub_error=0;long_queries_active=0;batch_index_initiate=0;batch_index_queue=0:0,0:0,0:0,0:0,0:0;batch_index_complete=0;batch_index_error=0;batch_index_timeout=0;batch_index_delay=0;batch_index_unused_buffers=0;batch_index_huge_buffers=0;batch_index_created_buffers=0;batch_index_destroyed_buffers=0;batch_index_proto_uncompressed_pct=0.000;batch_index_proto_compression_ratio=1.000;paxos_principal=BB9030011AC4202;time_since_rebalance=97888;migrate_allowed=true;migrate_partitions_remaining=0;fabric_bulk_send_rate=0;fabric_bulk_recv_rate=0;fabric_ctrl_send_rate=0;fabric_ctrl_recv_rate=0;fabric_meta_send_rate=0;fabric_meta_recv_rate=0;fabric_rw_send_rate=0;fabric_rw_recv_rate=0" - rawMetrics["build"] = "6.2.0.0-94-g2ccdbdd" - rawMetrics["service-clear-std"] = "172.17.0.3:3000" - rawMetrics["cluster-name"] = "null" - rawMetrics["get-stats:context=xdr;dc=backup_dc_asdev20"] = "lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;nodes=0;throughput=0;latency_ms=0;lap_us=12" - rawMetrics["raw_metrics_keys"] = "build:6.2.0.0-94-g2ccdbdd dcs=backup_dc_asdev20,a,b;src-id=0;trace-sample=0 namespaces:test;materials;employees;ns_test_1;ns_test_2;ns_test_3;ns_test_4;ns_test_5 sindex:ns=test:indexname=test_sindex1:set=test_set2:bin=occurred:type=numeric:indextype=default:context=null:state=RW" - rawMetrics["get-config:context=xdr"] = "dcs=backup_dc_asdev20,a,b;src-id=0;trace-sample=0" - rawMetrics["sindex"] = "ns=test:indexname=test_sindex1:set=test_set2:bin=occurred:type=numeric:indextype=default:context=null:state=RW" + // build, cluster-name, service-ip + rawMetrics["build"] = md.getBuild() + rawMetrics["cluster-name"] = md.getClusterName() + rawMetrics["service-clear-std"] = md.getServiceClearStd() + + // namespace + for _, entry := range md.namespaces_stats { + elements := strings.Split(entry, ":") + // format: namespace-stats:test:ns_cluster_size=1;effective_ ( 2nd element is the namespace name "test") + key := "namespace/" + elements[1] + rawMetrics[key] = elements[2] + } + + // node-stats & node-configs + for _, entry := range md.node_stats { + + // node-configs: & node-stats: + elements := strings.Split(entry, ":") + + if strings.HasPrefix(elements[0], "node-stats") { + key := "statistics" + rawMetrics[key] = elements[1] + } else if strings.HasPrefix(elements[0], "node-config") { + key := "get-config:context=service" + rawMetrics[key] = elements[1] + } + } + + // sindex-stats + for _, entry := range md.sindex_stats { + + // sindex-stats:test:test_sindex1: + elements := strings.Split(entry, ":") + + if strings.HasPrefix(elements[0], "sindex-stats") { + key := "sindex/" + elements[1] + "/" + elements[2] + rawMetrics[key] = elements[3] + } + } + + // xdr- (dc/namespace) (config/stats) + for _, entry := range md.xdr_stats { + + // sindex-stats:test:test_sindex1: + elements := strings.Split(entry, ":") + + if strings.HasPrefix(elements[0], "xdr-get-config") { + key := "get-config" + ":" + elements[1] + rawMetrics[key] = elements[2] + } else if strings.HasPrefix(elements[0], "xdr-get-stat") { + key := "get-stats" + ":" + elements[1] + rawMetrics[key] = elements[2] + } + } return rawMetrics } -func requestInfoNamespaces(rawMetrics map[string]string) map[string]string { +func (md *MockAerospikeServer) getBuild() string { + elements := strings.Split(md.passone_output_str, " ") + + for _, entry := range elements { + + if strings.HasPrefix(entry, "build") { + colonIndex := strings.Index(entry, ":") + // parts := strings.Split(entry, ":") + value := entry[colonIndex+1:] + return value + } + } + return "" +} + +func (md *MockAerospikeServer) getClusterName() string { + return md.cluster_name[0] +} + +func (md *MockAerospikeServer) getServiceClearStd() string { + return strings.Split(md.service_clear_std[0], "=")[1] +} + +func (md *MockAerospikeServer) createXdrPassOneKeys() map[string]string { + passOneOutput := make(map[string]string) + elements := strings.Split(md.passone_output_str, " ") + + for _, entry := range elements { + + if strings.HasPrefix(entry, "get-config:context=xdr") { + str := strings.ReplaceAll(entry, "get-config:context=xdr:", "") + passOneOutput["get-config:context=xdr"] = str + } else if strings.HasPrefix(entry, "get-stats:context=xdr") { + str := strings.ReplaceAll(entry, "get-stats:context=xdr:", "") + passOneOutput["get-stat:context=xdr"] = str + } + } + + passOneOutput["namespaces"] = md.passone_outputs_map["namespaces"] + + return passOneOutput + +} + +func (md *MockAerospikeServer) requestInfoNamespaces() map[string]string { pass2Metrics := make(map[string]string) - // pass2Metrics["namespaces"] = "test;bar" - lNamespaces := "" - for metricsGrpKey := range rawMetrics { - if strings.HasPrefix(metricsGrpKey, "namespace/") { - // our raw-metrics is of construct "namespace/", so split and take - ns := strings.Split(metricsGrpKey, "/")[1] - lNamespaces = ns + ";" + lNamespaces + namespaces := "" + for _, entry := range md.namespaces_stats { + elements := strings.Split(entry, ":") + // format: namespace-stats:test:ns_cluster_size=1;effective_ ( 2nd element is the namespace name "test") + ns := strings.TrimSpace(elements[1]) + if len(namespaces) > 0 { + namespaces = namespaces + ";" + ns + } else { + namespaces = ns } } - pass2Metrics["namespaces"] = strings.TrimSuffix(lNamespaces, ";") + + pass2Metrics["namespaces"] = strings.TrimSuffix(namespaces, ";") return pass2Metrics } -func createNamespacePassTwoExpectedOutputs(rawMetrics map[string]string) []string { +func (md *MockAerospikeServer) createNamespacePassTwoExpectedOutputs() []string { lNamespaces := []string{} + rawMetrics := md.fetchRawMetrics() + for metricsGrpKey := range rawMetrics { if strings.HasPrefix(metricsGrpKey, "namespace/") { - // our raw-metrics is of construct "namespace/", so split and take lNamespaces = append(lNamespaces, metricsGrpKey) } } return lNamespaces } -func copyConfigLabels() map[string]string { - cfgLabels := config.AeroProm.MetricLabels +func (md *MockAerospikeServer) createXdrPassTwoExpectedOutputs(passOneOutputs map[string]string) []string { + passTwoOutputs := []string{} + rawMetrics := md.fetchRawMetrics() - returnLabelMap := make(map[string]string) - for key, value := range cfgLabels { - returnLabelMap[key] = value + for k := range rawMetrics { + if strings.HasPrefix(k, "get-stats:") || strings.HasPrefix(k, "get-config:") { + passTwoOutputs = append(passTwoOutputs, k) + } } + // append namespaces - return returnLabelMap + return passTwoOutputs } -func createLabelByNames(labelsMap map[string]string) string { - - arr_label_names := []string{} - createdLabelString := "" +func (sim *MockAerospikeServer) createSindexPassTwoExpectedOutputs(mas *MockAerospikeServer) []string { + lSindexNames := []string{} + // rawMetricsKeys := rawMetrics["raw_metrics_keys"] + rawMetricsKeys := mas.fetchRawMetrics() - for key := range labelsMap { - arr_label_names = append(arr_label_names, strings.TrimSpace(key)) + for k := range rawMetricsKeys { + if strings.HasPrefix(k, "sindex/") { + lSindexNames = append(lSindexNames, k) + } } - // sort the keys - sort.Strings(arr_label_names) - - for idx := range arr_label_names { - keyName := arr_label_names[idx] - value := labelsMap[keyName] - createdLabelString = createdLabelString + constructLabelElement(keyName, strings.TrimSpace(value)) - } + return lSindexNames - return strings.TrimSpace(createdLabelString) } /* generated the output used by the Namespace-Watcher TestCases, the same output may not be suitable for other watchers, so for each data-context we need to have a method to generate expected output */ -func createNamespaceWatcherExpectedOutputs(nsName string, addNsToKey bool) (map[string][]string, map[string][]string) { +// Namespace related +type MockNamespacePromMetricGenerator struct { + // this will hold the golbal array/objects for Namespaces + lExpectedMetricNamedValues map[string][]string + lExpectedMetricLabels map[string][]string +} + +func (nsw *MockNamespacePromMetricGenerator) createNamespaceWatcherExpectedOutputs(mas *MockAerospikeServer, nsName string, addNsToKey bool) (map[string][]string, map[string][]string) { // regex to check if a stat is storage-engine seDynamicExtractor := regexp.MustCompile(`storage\-engine\.(?Pfile|device)\[(?P\d+)\]\.(?P.+)`) + itDynamicExtractor := regexp.MustCompile(`index\-type\.(?Pmount)\[(?P\d+)\]\.(?P.+)`) + sintDynamicExtractor := regexp.MustCompile(`sindex\-type\.(?Pmount)\[(?P\d+)\]\.(?P.+)`) - lExpectedMetricNamedValues := map[string][]string{} - lExpectedMetricLabels := map[string][]string{} + // + nsw.lExpectedMetricNamedValues = map[string][]string{} + nsw.lExpectedMetricLabels = map[string][]string{} - rawMetrics := getRawMetrics() + // rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() clusterName := rawMetrics["cluster-name"] service := rawMetrics["service-clear-std"] @@ -146,15 +321,24 @@ func createNamespaceWatcherExpectedOutputs(nsName string, addNsToKey bool) (map[ // reconvert float back to string value := fmt.Sprintf("%.0f", convertedValue) - match := seDynamicExtractor.FindStringSubmatch(key) - if len(match) != 4 { + // not a storage-engine or index-type or sindex-type + isNormalStat := !isStatArrayType(key) + + if isNormalStat { + + labelsMap := copyConfigLabels() + if strings.HasPrefix(key, "index-type") { + labelsMap["index"] = stats["index-type"] + } else if strings.HasPrefix(key, "sindex-type") { + labelsMap["sindex"] = stats["sindex-type"] + } key = strings.ReplaceAll(key, "-", "_") key = strings.ReplaceAll(key, ".", "_") metric := "aerospike_namespace_" + key - valuesArr := lExpectedMetricNamedValues[metric] - labelsArr := lExpectedMetricLabels[metric] + valuesArr := nsw.lExpectedMetricNamedValues[metric] + labelsArr := nsw.lExpectedMetricLabels[metric] if valuesArr == nil { valuesArr = []string{} } @@ -164,30 +348,62 @@ func createNamespaceWatcherExpectedOutputs(nsName string, addNsToKey bool) (map[ // labelStr := "[" + cfgLabelsToAdd + "name:\"cluster_name\" value:\"" + clusterName + "\" name:\"ns\" value:\"" + ns + "\" name:\"service\" value:\"" + service + "\" ]" - labelsMap := copyConfigLabels() labelsMap["ns"] = ns labelsMap["service"] = service labelsMap["cluster_name"] = clusterName - sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]" + labelByNames := createLabelByNames(labelsMap) + sorted_label_str := "[" + labelByNames + " ]" valuesArr = append(valuesArr, value) labelsArr = append(labelsArr, sorted_label_str) - mapKeyname := makeKeyname(nsName, metric, addNsToKey) - lExpectedMetricNamedValues[mapKeyname] = valuesArr - lExpectedMetricLabels[mapKeyname] = labelsArr + labelString := stringifyLabel(labelByNames) + + // key will be like /namespace/, this we use this check during assertion + mapKeyname := makeKeyname(nsName, labelString, true) + mapKeyname = makeKeyname(mapKeyname, metric, true) + nsw.lExpectedMetricNamedValues[mapKeyname] = valuesArr + nsw.lExpectedMetricLabels[mapKeyname] = labelsArr } else { - metricType := match[1] - metricIndex := match[2] - metricName := match[3] - deviceOrFileName := stats["storage-engine."+metricType+"["+metricIndex+"]"] + labelsMap := copyConfigLabels() - metric := "aerospike_namespace_storage_engine_" + metricType + "_" + metricName + metricType := "" + metricIndex := "" + metricName := "" + deviceOrFileName := "" + metric := "" + + if strings.HasPrefix(key, "storage-engine") { + match := seDynamicExtractor.FindStringSubmatch(key) + metricType = match[1] + metricIndex = match[2] + metricName = match[3] + deviceOrFileName = stats["storage-engine."+metricType+"["+metricIndex+"]"] + metric = "aerospike_namespace_storage_engine_" + metricType + "_" + metricName + + } else if strings.HasPrefix(key, "index-type") { + match := itDynamicExtractor.FindStringSubmatch(key) + metricType = match[1] + metricIndex = match[2] + metricName = match[3] + deviceOrFileName = stats["index-type."+metricType+"["+metricIndex+"]"] + + metric = "aerospike_namespace_index_type_" + metricType + "_" + metricName + labelsMap["index"] = stats["index-type"] + } else if strings.HasPrefix(key, "sindex-type") { + match := sintDynamicExtractor.FindStringSubmatch(key) + metricType = match[1] + metricIndex = match[2] + metricName = match[3] + deviceOrFileName = stats["sindex-type."+metricType+"["+metricIndex+"]"] + metric = "aerospike_namespace_sindex_type_" + metricType + "_" + metricName + labelsMap["sindex"] = stats["sindex-type"] + } - valuesArr := lExpectedMetricNamedValues[metric] - labelsArr := lExpectedMetricLabels[metric] + valuesArr := nsw.lExpectedMetricNamedValues[metric] + labelsArr := nsw.lExpectedMetricLabels[metric] if valuesArr == nil { valuesArr = []string{} } @@ -197,40 +413,65 @@ func createNamespaceWatcherExpectedOutputs(nsName string, addNsToKey bool) (map[ // labelStr := "[" + cfgLabelsToAdd + "name:\"cluster_name\" value:\"" + clusterName + "\" name:\"file\" value:\"" + deviceOrFileName + "\" name:\"file_index\" value:\"" + metricIndex + "\" name:\"ns\" value:\"" + ns + "\" name:\"service\" value:\"" + service + "\" ]" - labelsMap := copyConfigLabels() labelsMap["cluster_name"] = clusterName - labelsMap["file"] = deviceOrFileName + labelsMap[metricType] = deviceOrFileName labelsMap["ns"] = ns - labelsMap["file_index"] = metricIndex + labelsMap[metricType+"_index"] = metricIndex labelsMap["service"] = service - sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]" + labelByNames := createLabelByNames(labelsMap) + sorted_label_str := "[" + labelByNames + " ]" valuesArr = append(valuesArr, value) labelsArr = append(labelsArr, sorted_label_str) - mapKeyname := makeKeyname(nsName, metric, addNsToKey) + labelString := stringifyLabel(labelByNames) + + // key will be like /namespace/, this we use this check during assertion + mapKeyname := makeKeyname(nsName, labelString, true) + mapKeyname = makeKeyname(mapKeyname, metric, true) + + nsw.lExpectedMetricNamedValues[mapKeyname] = valuesArr + nsw.lExpectedMetricLabels[mapKeyname] = labelsArr - lExpectedMetricNamedValues[mapKeyname] = valuesArr - lExpectedMetricLabels[mapKeyname] = labelsArr } } } } - return lExpectedMetricNamedValues, lExpectedMetricLabels + return nsw.lExpectedMetricNamedValues, nsw.lExpectedMetricLabels } -func createNodeStatsWatcherExpectedOutputs(serviceIp string) (map[string][]string, map[string][]string) { - lExpectedMetricNamedValues := map[string][]string{} - lExpectedMetricLabels := map[string][]string{} +func isStatArrayType(statToProcess string) bool { + if strings.HasPrefix(statToProcess, INDEX_TYPE) && strings.Contains(statToProcess, "[") { + return true + } else if strings.HasPrefix(statToProcess, SINDEX_TYPE) && strings.Contains(statToProcess, "[") { + return true + } else if strings.HasPrefix(statToProcess, STORAGE_ENGINE) && strings.Contains(statToProcess, "[") { + return true + } - rawMetrics := getRawMetrics() + return false +} + +// Node-stats related +type MockNodestatPromMetricGenerator struct { + // this will hold the golbal array/objects for Node-stats + lExpectedMetricNamedValues map[string][]string + lExpectedMetricLabels map[string][]string +} + +func (nst *MockNodestatPromMetricGenerator) createNodeStatsWatcherExpectedOutputs(mas *MockAerospikeServer, serviceIp string) (map[string][]string, map[string][]string) { + nst.lExpectedMetricNamedValues = map[string][]string{} + nst.lExpectedMetricLabels = map[string][]string{} + + // rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() clusterName := rawMetrics["cluster-name"] service := rawMetrics["service-clear-std"] for metricsGrpKey := range rawMetrics { - if strings.HasPrefix(metricsGrpKey, "statistics") { + if strings.HasPrefix(metricsGrpKey, "statistics") || strings.HasPrefix(metricsGrpKey, "get-config:context=service") { grpValues := rawMetrics[metricsGrpKey] stats := splitAndRetrieveStats(grpValues, ";") for s, v := range stats { @@ -256,8 +497,8 @@ func createNodeStatsWatcherExpectedOutputs(serviceIp string) (map[string][]strin key = strings.ReplaceAll(key, "-", "_") key = strings.ReplaceAll(key, ".", "_") metric := "aerospike_node_stats_" + key - valuesArr := lExpectedMetricNamedValues[metric] - labelsArr := lExpectedMetricLabels[metric] + valuesArr := nst.lExpectedMetricNamedValues[metric] + labelsArr := nst.lExpectedMetricLabels[metric] if valuesArr == nil { valuesArr = []string{} @@ -277,21 +518,30 @@ func createNodeStatsWatcherExpectedOutputs(serviceIp string) (map[string][]strin labelsArr = append(labelsArr, sorted_label_str) mapKeyname := makeKeyname(serviceIp, metric, true) - lExpectedMetricNamedValues[mapKeyname] = valuesArr - lExpectedMetricLabels[mapKeyname] = labelsArr + nst.lExpectedMetricNamedValues[mapKeyname] = valuesArr + nst.lExpectedMetricLabels[mapKeyname] = labelsArr } } } - return lExpectedMetricNamedValues, lExpectedMetricLabels + return nst.lExpectedMetricNamedValues, nst.lExpectedMetricLabels } -func createSetsWatcherExpectedOutputs(t string) (map[string][]string, map[string][]string) { - lExpectedMetricNamedValues := map[string][]string{} - lExpectedMetricLabels := map[string][]string{} +// Set related +type MockSetsPromMetricGenerator struct { + // this will hold the golbal array/objects for Sets + lExpectedMetricNamedValues map[string][]string + lExpectedMetricLabels map[string][]string +} + +func (msdg *MockSetsPromMetricGenerator) createSetsWatcherExpectedOutputs(mas *MockAerospikeServer, setName string) (map[string][]string, map[string][]string) { + msdg.lExpectedMetricNamedValues = map[string][]string{} + msdg.lExpectedMetricLabels = map[string][]string{} + + // rawMetrics := getRawMetrics() - rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() clusterName := rawMetrics["cluster-name"] service := rawMetrics["service-clear-std"] @@ -330,8 +580,8 @@ func createSetsWatcherExpectedOutputs(t string) (map[string][]string, map[string key = strings.ReplaceAll(key, "-", "_") key = strings.ReplaceAll(key, ".", "_") metric := "aerospike_sets_" + key - valuesArr := lExpectedMetricNamedValues[metric] - labelsArr := lExpectedMetricLabels[metric] + valuesArr := msdg.lExpectedMetricNamedValues[metric] + labelsArr := msdg.lExpectedMetricLabels[metric] if valuesArr == nil { valuesArr = []string{} @@ -356,27 +606,35 @@ func createSetsWatcherExpectedOutputs(t string) (map[string][]string, map[string valuesArr = append(valuesArr, value) labelsArr = append(labelsArr, sorted_label_str) - lExpectedMetricNamedValues[mapKeyname] = valuesArr - lExpectedMetricLabels[mapKeyname] = labelsArr + msdg.lExpectedMetricNamedValues[mapKeyname] = valuesArr + msdg.lExpectedMetricLabels[mapKeyname] = labelsArr } } } } - return lExpectedMetricNamedValues, lExpectedMetricLabels + return msdg.lExpectedMetricNamedValues, msdg.lExpectedMetricLabels } -func createLatencysWatcherExpectedOutputs(namespaceWithSetName string) (map[string][]string, map[string][]string) { - lExpectedMetricNamedValues := map[string][]string{} - lExpectedMetricLabels := map[string][]string{} +// Latency related +type MockLatencyPromMetricGenerator struct { + // this will hold the golbal array/objects for Latencies + lExpectedMetricNamedValues map[string][]string + lExpectedMetricLabels map[string][]string +} + +func (ltc *MockLatencyPromMetricGenerator) createLatencysWatcherExpectedOutputs(mas *MockAerospikeServer, namespaceWithSetName string) (map[string][]string, map[string][]string) { + ltc.lExpectedMetricNamedValues = map[string][]string{} + ltc.lExpectedMetricLabels = map[string][]string{} - rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() clusterName := rawMetrics["cluster-name"] service := rawMetrics["service-clear-std"] latencyLabelGroups := []string{"0", "+Inf", "1", "2", "4", "8", "16", "32", "64", "128", "256", "512", "1024", "2048", "4096", "8192", "16384", "32768", "65536"} + for metricsGrpKey := range rawMetrics { if strings.HasPrefix(metricsGrpKey, "latencies") { grpValues := rawMetrics[metricsGrpKey] @@ -398,8 +656,8 @@ func createLatencysWatcherExpectedOutputs(namespaceWithSetName string) (map[stri count_metric := "aerospike_latencies" + "_" + operation + "_" + "ms" + "_" + "count" mapKeyname := service + "_" + namespace + "_" + operation - cntValuesArr := lExpectedMetricNamedValues[count_metric] - cntLabelsArr := lExpectedMetricLabels[count_metric] + cntValuesArr := ltc.lExpectedMetricNamedValues[count_metric] + cntLabelsArr := ltc.lExpectedMetricLabels[count_metric] // Add latency-metric-COUNT if cntValuesArr == nil { @@ -419,8 +677,8 @@ func createLatencysWatcherExpectedOutputs(namespaceWithSetName string) (map[stri cntValuesArr = append(cntValuesArr, allLatencyValues[1]) cntLabelsArr = append(cntLabelsArr, sorted_label_str) - lExpectedMetricNamedValues[mapKeyname] = cntValuesArr - lExpectedMetricLabels[mapKeyname] = cntLabelsArr + ltc.lExpectedMetricNamedValues[mapKeyname] = cntValuesArr + ltc.lExpectedMetricLabels[mapKeyname] = cntLabelsArr // Add latency-metric-BUCKET with le for idx := range allLatencyValues { @@ -429,8 +687,8 @@ func createLatencysWatcherExpectedOutputs(namespaceWithSetName string) (map[stri value := allLatencyValues[idx] mapKeyname := service + "_" + namespace + "_" + operation + "_" + strings.ReplaceAll(le, "+", "") - bktValuesArr := lExpectedMetricNamedValues[bucket_metric] - bktLabelsArr := lExpectedMetricLabels[bucket_metric] + bktValuesArr := ltc.lExpectedMetricNamedValues[bucket_metric] + bktLabelsArr := ltc.lExpectedMetricLabels[bucket_metric] // Add latency-metric-COUNT if bktValuesArr == nil { @@ -451,8 +709,8 @@ func createLatencysWatcherExpectedOutputs(namespaceWithSetName string) (map[stri bktValuesArr = append(bktValuesArr, value) bktLabelsArr = append(bktLabelsArr, sorted_label_str) - lExpectedMetricNamedValues[mapKeyname] = bktValuesArr - lExpectedMetricLabels[mapKeyname] = bktLabelsArr + ltc.lExpectedMetricNamedValues[mapKeyname] = bktValuesArr + ltc.lExpectedMetricLabels[mapKeyname] = bktLabelsArr } @@ -466,22 +724,29 @@ func createLatencysWatcherExpectedOutputs(namespaceWithSetName string) (map[stri } } - return lExpectedMetricNamedValues, lExpectedMetricLabels + return ltc.lExpectedMetricNamedValues, ltc.lExpectedMetricLabels } // Xdr related +type MockXdrPromMetricGenerator struct { + // this will hold the golbal array/objects for Xdr + lExpectedMetricNamedValues map[string][]string + lExpectedMetricLabels map[string][]string +} -func createXdrsWatcherExpectedOutputs(t string) (map[string][]string, map[string][]string) { - lExpectedMetricNamedValues := map[string][]string{} - lExpectedMetricLabels := map[string][]string{} +func (xdr *MockXdrPromMetricGenerator) createXdrsWatcherExpectedOutputs(mas *MockAerospikeServer, xdrDcName string) (map[string][]string, map[string][]string) { - rawMetrics := getRawMetrics() + // re-initialize whenever called + xdr.lExpectedMetricNamedValues = make(map[string][]string) + xdr.lExpectedMetricLabels = make(map[string][]string) - clusterName := rawMetrics["cluster-name"] - service := rawMetrics["service-clear-std"] + rawMetrics := mas.fetchRawMetrics() + + clusterName := mas.getClusterName() + service := mas.getServiceClearStd() for metricsGrpKey := range rawMetrics { - if strings.HasPrefix(metricsGrpKey, "get-stats:context=xdr;dc=") { + if strings.HasPrefix(metricsGrpKey, "get-stats:context=xdr;dc=") || strings.HasPrefix(metricsGrpKey, "get-configs:context=xdr;dc=") { grpValues := rawMetrics[metricsGrpKey] xdrDcInfos := strings.Split(grpValues, ";") for idx := range xdrDcInfos { @@ -490,7 +755,12 @@ func createXdrsWatcherExpectedOutputs(t string) (map[string][]string, map[string stats := splitAndRetrieveStats(singleXdrDcValues, ":") - dcName := strings.ReplaceAll(metricsGrpKey, "get-stats:context=xdr;dc=", "") + // get-config:context=xdr;dc=backup_dc_asdev20_second;namespace=bar_device + // get-config:context=xdr;dc=backup_dc_asdev20_second + // get-stats:context=xdr;dc=backup_dc_asdev20_second;namespace=bar_device + // get-stats:context=xdr;dc=backup_dc_asdev20_second + // grpSplitElements := strings.Split(metricsGrpKey, ";") + // dcName := strings.Split(grpSplitElements[1], "=")[1] for s, v := range stats { key := s @@ -514,9 +784,13 @@ func createXdrsWatcherExpectedOutputs(t string) (map[string][]string, map[string key = strings.ReplaceAll(key, "-", "_") key = strings.ReplaceAll(key, ".", "_") - metric := "aerospike_xdr" + "_" + key - valuesArr := lExpectedMetricNamedValues[metric] - labelsArr := lExpectedMetricLabels[metric] + + // metric := "aerospike_xdr" + "_" + key + // namespace := "no-namespace" // just to represent this metric is only at DC level + dcName, namespace, metric := xdr.constructXdrMetricname(metricsGrpKey, key) + + valuesArr := xdr.lExpectedMetricNamedValues[metric] + labelsArr := xdr.lExpectedMetricLabels[metric] if valuesArr == nil { valuesArr = []string{} @@ -526,9 +800,6 @@ func createXdrsWatcherExpectedOutputs(t string) (map[string][]string, map[string } // [name:"cluster_name" value:"null" name:"dc" value:"backup_dc_asdev20" name:"service" value:"172.17.0.3:3000" ] - mapKeyname := makeKeyname(dcName, metric, true) - mapKeyname = makeKeyname(service, mapKeyname, true) - // labelStr := "[" + cfgLabelsToAdd + "name:\"cluster_name\" value:\"" + clusterName + "\" name:\"ns\" value:\"" + namespace + "\" name:\"service\" value:\"" + service + "\" name:\"set\" value:\"" + setName + "\"" + " ]" labelsMap := copyConfigLabels() @@ -536,86 +807,69 @@ func createXdrsWatcherExpectedOutputs(t string) (map[string][]string, map[string labelsMap["service"] = service labelsMap["cluster_name"] = clusterName + if strings.Contains(metricsGrpKey, "namespace") { + grpSplitElements := strings.Split(metricsGrpKey, ";") + // get-config:context=xdr;dc=backup_dc_asdev20_second;namespace=bar_device + namespace = strings.Split(grpSplitElements[2], "=")[1] + labelsMap["ns"] = namespace + } + sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]" valuesArr = append(valuesArr, value) labelsArr = append(labelsArr, sorted_label_str) - lExpectedMetricNamedValues[mapKeyname] = valuesArr - lExpectedMetricLabels[mapKeyname] = labelsArr + mapKeyname := makeKeyname(dcName, metric, true) + mapKeyname = makeKeyname(namespace, mapKeyname, true) + mapKeyname = makeKeyname(service, mapKeyname, true) + + xdr.lExpectedMetricNamedValues[mapKeyname] = valuesArr + xdr.lExpectedMetricLabels[mapKeyname] = labelsArr } // end stats for-loop } // end xdrDcInfos } } - return lExpectedMetricNamedValues, lExpectedMetricLabels + return xdr.lExpectedMetricNamedValues, xdr.lExpectedMetricLabels } -func createXdrPassTwoExpectedOutputs(rawMetrics map[string]string) []string { - lXdrDc := []string{} - rawMetricsKeys := rawMetrics["raw_metrics_keys"] - contextKeys := strings.Split(rawMetricsKeys, " ") - - for idx := range contextKeys { +func (xdr *MockXdrPromMetricGenerator) constructXdrMetricname(infoKeyToProcess string, stat string) (string, string, string) { - metricsGrpKeys := contextKeys[idx] + kvInfoKeyToProcess := parseStats(infoKeyToProcess, ";") + _, cfgOk := kvInfoKeyToProcess["get-config:context"] + _, statOk := kvInfoKeyToProcess["get-stats:context"] + dcName := kvInfoKeyToProcess["dc"] + nsName, nsOk := kvInfoKeyToProcess["namespace"] - if strings.HasPrefix(metricsGrpKeys, "dcs=") { + // either this is a config key or a stat having namespace (both are new use-cases) hence handle here - // we are assuming only 1 xdr dc for now - elements := strings.Split(metricsGrpKeys, ";") + if cfgOk && nsOk { + return dcName, nsName, ("aerospike_xdr" + "_" + "dc_namespace_" + stat) + } else if statOk && nsOk { + return dcName, nsName, ("aerospike_xdr" + "_" + "dc_namespace_" + stat) + } else if cfgOk { + return dcName, nsName, ("aerospike_xdr" + "_" + "dc_" + stat) + } - xdr_dcs_str := elements[0] - xdr_dcs_str = strings.ReplaceAll(xdr_dcs_str, "dcs=", "") - dcs := strings.Split(xdr_dcs_str, ",") + return dcName, "no-namespace", "aerospike_xdr" + "_" + stat // no-prefix/default i.e. no suffix like "dc" / "dc_namespace" - for i := range dcs { - lXdrDc = append(lXdrDc, "get-stats:context=xdr;dc="+dcs[i]) - } - } - - } - return lXdrDc } // Sindex related methods -type MockSindexDataGen struct { +type MockSindexPromMetricGenerator struct { // this will hold the golbal array/objects for Sindex lExpectedMetricNamedValues map[string][]string lExpectedMetricLabels map[string][]string } -func (sim *MockSindexDataGen) createSindexPassTwoExpectedOutputs(rawMetrics map[string]string) []string { - lSindexNames := []string{} - rawMetricsKeys := rawMetrics["raw_metrics_keys"] - contextKeys := strings.Split(rawMetricsKeys, " ") - - for idx := range contextKeys { - - metricsGrpKeys := contextKeys[idx] - - if strings.HasPrefix(metricsGrpKeys, "sindex") { - - // we are assuming only 1 xdr dc for now - elements := splitAndRetrieveStats(metricsGrpKeys, ":") - - // in mock assuming only 1 Sindex - lSindexNames = append(lSindexNames, "sindex/"+elements["ns"]+"/"+elements["indexname"]) - } - - } - - return lSindexNames - -} - -func (sim *MockSindexDataGen) createSindexWatcherTestData() (map[string][]string, map[string][]string) { +func (sim *MockSindexPromMetricGenerator) createSindexWatcherTestData(mas *MockAerospikeServer) (map[string][]string, map[string][]string) { // re-initialize whenever called sim.lExpectedMetricNamedValues = make(map[string][]string) sim.lExpectedMetricLabels = make(map[string][]string) - rawMetrics := getRawMetrics() + // rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() clusterName := rawMetrics["cluster-name"] service := rawMetrics["service-clear-std"] @@ -685,19 +939,19 @@ func (sim *MockSindexDataGen) createSindexWatcherTestData() (map[string][]string return sim.lExpectedMetricNamedValues, sim.lExpectedMetricLabels } -func (siMock *MockSindexDataGen) extractNamespaceSetSindexname(sindexKey string) (string, string) { +func (siMock *MockSindexPromMetricGenerator) extractNamespaceSetSindexname(sindexKey string) (string, string) { elements := strings.Split(sindexKey, "/") return elements[1], elements[2] } // User related utilities -type MockUsersDataGen struct { +type MockUsersPromMetricGenerator struct { lExpectedMetricNamedValues map[string][]string lExpectedMetricLabels map[string][]string } -func (mockUsers *MockUsersDataGen) createDummyUserRoles() []*aero.UserRoles { +func (mockUsers *MockUsersPromMetricGenerator) createDummyUserRoles() []*aero.UserRoles { userRoles := []*aero.UserRoles{} MAX_USERS := 4 @@ -727,14 +981,14 @@ func (mockUsers *MockUsersDataGen) createDummyUserRoles() []*aero.UserRoles { return userRoles } -func (mockUsers *MockUsersDataGen) createMockUserData() (map[string][]string, map[string][]string) { +func (mockUsers *MockUsersPromMetricGenerator) createMockUserData(mas *MockAerospikeServer) (map[string][]string, map[string][]string) { userRoles := mockUsers.createDummyUserRoles() // re-initialize whenever called mockUsers.lExpectedMetricNamedValues = make(map[string][]string) mockUsers.lExpectedMetricLabels = make(map[string][]string) - rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() clusterName := rawMetrics["cluster-name"] service := rawMetrics["service-clear-std"] @@ -858,7 +1112,7 @@ func (mockUsers *MockUsersDataGen) createMockUserData() (map[string][]string, ma return mockUsers.lExpectedMetricNamedValues, mockUsers.lExpectedMetricLabels } -func (mockUsers *MockUsersDataGen) isUserAllowed(username string) bool { +func (mockUsers *MockUsersPromMetricGenerator) isUserAllowed(username string) bool { userAllowlist := config.Aerospike.UserMetricsUsersAllowlist userBlocklist := config.Aerospike.UserMetricsUsersBlocklist if len(userBlocklist) == 0 && len(userAllowlist) == 0 { diff --git a/observer.go b/observer.go index 1e9511ef..1a4dff06 100644 --- a/observer.go +++ b/observer.go @@ -231,9 +231,13 @@ func (o *Observer) refresh(ch chan<- prometheus.Metric) (map[string]string, erro infoKeys = append(infoKeys, keys...) } } + // append infoKey "build" - this is removed from WatcherLatencies to avoid forced watcher sequence during refresh + infoKeys = append(infoKeys, "build") - // info request for first set of info keys - rawMetrics, err := o.requestInfo(retryCount, infoKeys) + // info request for first set of info keys, this retrives configs from server + // from namespaces,server/node-stats, xdr + // if for any context (like jobs, latencies etc.,) no configs, they are not sent to server + passOneOutput, err := o.requestInfo(retryCount, infoKeys) if err != nil { return nil, err } @@ -242,25 +246,24 @@ func (o *Observer) refresh(ch chan<- prometheus.Metric) (map[string]string, erro infoKeys = []string{ikClusterName, ikService, ikBuild} watcherInfoKeys := make([][]string, len(o.watchers)) for i, c := range o.watchers { - if keys := c.passTwoKeys(rawMetrics); len(keys) > 0 { + if keys := c.passTwoKeys(passOneOutput); len(keys) > 0 { infoKeys = append(infoKeys, keys...) watcherInfoKeys[i] = keys } } - // info request for second set of info keys - nRawMetrics, err := o.requestInfo(retryCount, infoKeys) + // info request for second set of info keys, this retrieves all the stats from server + rawMetrics, err := o.requestInfo(retryCount, infoKeys) if err != nil { - return rawMetrics, err + return passOneOutput, err } - rawMetrics = nRawMetrics - // sanitize the utf8 strings before sending them to watchers for k, v := range rawMetrics { rawMetrics[k] = sanitizeUTF8(v) } + // sanitize the utf8 strings before sending them to watchers for i, c := range o.watchers { if err := c.refresh(o, watcherInfoKeys[i], rawMetrics, ch); err != nil { return rawMetrics, err diff --git a/tests/mock_test_data.txt b/tests/mock_test_data.txt new file mode 100644 index 00000000..57e55b80 --- /dev/null +++ b/tests/mock_test_data.txt @@ -0,0 +1,56 @@ +build=6.3 +service-clear-std=172.17.0.3:3000 +cluster-name=63flash +passone_output:build:6.4.0.0-rc4 get-config:context=xdr:dcs=backup_dc_asdev20,backup_dc_asdev20_second;src-id=0;trace-sample=0 namespaces:test;bar_device;materials;ns_test_on_flash;test_on_shmem;bar_on_flash;pmkohl_on_device sindex:ns=test:indexname=test_sindex1:set=from_branch_2:bin=occurred:type=numeric:indextype=default:context=null:state=RW + +namespace-stats:test:ns_cluster_size=1;effective_replication_factor=1;objects=3136757;tombstones=0;xdr_tombstones=0;xdr_bin_cemeteries=0;master_objects=3136762;master_tombstones=0;prole_objects=0;prole_tombstones=0;non_replica_objects=0;non_replica_tombstones=0;unreplicated_records=0;dead_partitions=0;unavailable_partitions=0;clock_skew_stop_writes=false;stop_writes=false;hwm_breached=false;current_time=427402124;non_expirable_objects=0;expired_objects=0;evicted_objects=0;evict_ttl=0;evict_void_time=0;smd_evict_void_time=0;nsup_cycle_duration=0;nsup_cycle_deleted_pct=0.00;truncate_lut=0;truncating=false;sindex_gc_cleaned=0;memory_used_bytes=433323819;memory_used_data_bytes=215793707;memory_used_index_bytes=200752896;memory_used_set_index_bytes=0;memory_used_sindex_bytes=16777216;memory_free_pct=89;xmem_id=1;available_bin_names=65532;record_proto_uncompressed_pct=0.000;record_proto_compression_ratio=1.000;query_proto_uncompressed_pct=0.000;query_proto_compression_ratio=1.000;pending_quiesce=false;effective_is_quiesced=false;nodes_quiesced=0;effective_prefer_uniform_balance=true;migrate_tx_partitions_imbalance=0;migrate_tx_instances=0;migrate_rx_instances=0;migrate_tx_partitions_active=0;migrate_rx_partitions_active=0;migrate_tx_partitions_initial=0;migrate_tx_partitions_remaining=0;migrate_tx_partitions_lead_remaining=0;migrate_rx_partitions_initial=0;migrate_rx_partitions_remaining=0;migrate_records_skipped=0;migrate_records_transmitted=0;migrate_record_retransmits=0;migrate_record_receives=0;migrate_signals_active=0;migrate_signals_remaining=0;appeals_tx_active=0;appeals_rx_active=0;appeals_tx_remaining=0;appeals_records_exonerated=0;client_tsvc_error=0;client_tsvc_timeout=0;client_proxy_complete=0;client_proxy_error=0;client_proxy_timeout=0;client_read_success=0;client_read_error=0;client_read_timeout=0;client_read_not_found=0;client_read_filtered_out=0;client_write_success=4353140;client_write_error=0;client_write_timeout=0;client_write_filtered_out=0;xdr_client_write_success=0;xdr_client_write_error=0;xdr_client_write_timeout=0;client_delete_success=0;client_delete_error=0;client_delete_timeout=0;client_delete_not_found=0;client_delete_filtered_out=0;xdr_client_delete_success=0;xdr_client_delete_error=0;xdr_client_delete_timeout=0;xdr_client_delete_not_found=0;client_udf_complete=0;client_udf_error=0;client_udf_timeout=0;client_udf_filtered_out=0;client_lang_read_success=0;client_lang_write_success=0;client_lang_delete_success=0;client_lang_error=0;from_proxy_tsvc_error=0;from_proxy_tsvc_timeout=0;from_proxy_read_success=0;from_proxy_read_error=0;from_proxy_read_timeout=0;from_proxy_read_not_found=0;from_proxy_read_filtered_out=0;from_proxy_write_success=0;from_proxy_write_error=0;from_proxy_write_timeout=0;from_proxy_write_filtered_out=0;xdr_from_proxy_write_success=0;xdr_from_proxy_write_error=0;xdr_from_proxy_write_timeout=0;from_proxy_delete_success=0;from_proxy_delete_error=0;from_proxy_delete_timeout=0;from_proxy_delete_not_found=0;from_proxy_delete_filtered_out=0;xdr_from_proxy_delete_success=0;xdr_from_proxy_delete_error=0;xdr_from_proxy_delete_timeout=0;xdr_from_proxy_delete_not_found=0;from_proxy_udf_complete=0;from_proxy_udf_error=0;from_proxy_udf_timeout=0;from_proxy_udf_filtered_out=0;from_proxy_lang_read_success=0;from_proxy_lang_write_success=0;from_proxy_lang_delete_success=0;from_proxy_lang_error=0;batch_sub_tsvc_error=0;batch_sub_tsvc_timeout=0;batch_sub_proxy_complete=0;batch_sub_proxy_error=0;batch_sub_proxy_timeout=0;batch_sub_read_success=0;batch_sub_read_error=0;batch_sub_read_timeout=0;batch_sub_read_not_found=0;batch_sub_read_filtered_out=0;batch_sub_write_success=0;batch_sub_write_error=0;batch_sub_write_timeout=0;batch_sub_write_filtered_out=0;batch_sub_delete_success=0;batch_sub_delete_error=0;batch_sub_delete_timeout=0;batch_sub_delete_not_found=0;batch_sub_delete_filtered_out=0;batch_sub_udf_complete=0;batch_sub_udf_error=0;batch_sub_udf_timeout=0;batch_sub_udf_filtered_out=0;batch_sub_lang_read_success=0;batch_sub_lang_write_success=0;batch_sub_lang_delete_success=0;batch_sub_lang_error=0;from_proxy_batch_sub_tsvc_error=0;from_proxy_batch_sub_tsvc_timeout=0;from_proxy_batch_sub_read_success=0;from_proxy_batch_sub_read_error=0;from_proxy_batch_sub_read_timeout=0;from_proxy_batch_sub_read_not_found=0;from_proxy_batch_sub_read_filtered_out=0;from_proxy_batch_sub_write_success=0;from_proxy_batch_sub_write_error=0;from_proxy_batch_sub_write_timeout=0;from_proxy_batch_sub_write_filtered_out=0;from_proxy_batch_sub_delete_success=0;from_proxy_batch_sub_delete_error=0;from_proxy_batch_sub_delete_timeout=0;from_proxy_batch_sub_delete_not_found=0;from_proxy_batch_sub_delete_filtered_out=0;from_proxy_batch_sub_udf_complete=0;from_proxy_batch_sub_udf_error=0;from_proxy_batch_sub_udf_timeout=0;from_proxy_batch_sub_udf_filtered_out=0;from_proxy_batch_sub_lang_read_success=0;from_proxy_batch_sub_lang_write_success=0;from_proxy_batch_sub_lang_delete_success=0;from_proxy_batch_sub_lang_error=0;udf_sub_tsvc_error=0;udf_sub_tsvc_timeout=0;udf_sub_udf_complete=0;udf_sub_udf_error=0;udf_sub_udf_timeout=0;udf_sub_udf_filtered_out=0;udf_sub_lang_read_success=0;udf_sub_lang_write_success=0;udf_sub_lang_delete_success=0;udf_sub_lang_error=0;ops_sub_tsvc_error=0;ops_sub_tsvc_timeout=0;ops_sub_write_success=0;ops_sub_write_error=0;ops_sub_write_timeout=0;ops_sub_write_filtered_out=0;dup_res_ask=0;dup_res_respond_read=0;dup_res_respond_no_read=0;retransmit_all_read_dup_res=0;retransmit_all_write_dup_res=0;retransmit_all_write_repl_write=0;retransmit_all_delete_dup_res=0;retransmit_all_delete_repl_write=0;retransmit_all_udf_dup_res=0;retransmit_all_udf_repl_write=0;retransmit_all_batch_sub_dup_res=0;retransmit_udf_sub_dup_res=0;retransmit_udf_sub_repl_write=0;retransmit_ops_sub_dup_res=0;retransmit_ops_sub_repl_write=0;pi_query_short_basic_complete=0;pi_query_short_basic_error=0;pi_query_short_basic_timeout=0;pi_query_long_basic_complete=0;pi_query_long_basic_error=0;pi_query_long_basic_abort=0;pi_query_aggr_complete=0;pi_query_aggr_error=0;pi_query_aggr_abort=0;pi_query_udf_bg_complete=0;pi_query_udf_bg_error=0;pi_query_udf_bg_abort=0;pi_query_ops_bg_complete=0;pi_query_ops_bg_error=0;pi_query_ops_bg_abort=0;si_query_short_basic_complete=0;si_query_short_basic_error=0;si_query_short_basic_timeout=0;si_query_long_basic_complete=0;si_query_long_basic_error=0;si_query_long_basic_abort=0;si_query_aggr_complete=0;si_query_aggr_error=0;si_query_aggr_abort=0;si_query_udf_bg_complete=0;si_query_udf_bg_error=0;si_query_udf_bg_abort=0;si_query_ops_bg_complete=0;si_query_ops_bg_error=0;si_query_ops_bg_abort=0;geo_region_query_reqs=0;geo_region_query_cells=0;geo_region_query_points=0;geo_region_query_falsepos=0;re_repl_tsvc_error=0;re_repl_tsvc_timeout=0;re_repl_success=0;re_repl_error=0;re_repl_timeout=0;fail_xdr_forbidden=0;fail_key_busy=0;fail_generation=0;fail_record_too_big=0;fail_client_lost_conflict=0;fail_xdr_lost_conflict=0;deleted_last_bin=0;allow-ttl-without-nsup=false;background-query-max-rps=10000;conflict-resolution-policy=generation;conflict-resolve-writes=false;default-ttl=0;disable-cold-start-eviction=false;disable-write-dup-res=false;disallow-expunge=false;disallow-null-setname=false;enable-benchmarks-batch-sub=false;enable-benchmarks-ops-sub=false;enable-benchmarks-read=false;enable-benchmarks-udf=false;enable-benchmarks-udf-sub=false;enable-benchmarks-write=false;enable-hist-proxy=false;evict-hist-buckets=10000;evict-tenths-pct=5;force-long-queries=false;high-water-disk-pct=0;high-water-memory-pct=0;ignore-migrate-fill-delay=false;index-stage-size=1073741824;inline-short-queries=false;max-record-size=0;memory-size=4294967296;migrate-order=5;migrate-retransmit-ms=5000;migrate-sleep=1;nsup-hist-period=3600;nsup-period=0;nsup-threads=1;partition-tree-sprigs=256;prefer-uniform-balance=true;rack-id=0;read-consistency-level-override=off;reject-non-xdr-writes=false;reject-xdr-writes=false;replication-factor=2;sindex-stage-size=1073741824;single-query-threads=4;stop-writes-pct=90;stop-writes-sys-memory-pct=90;strong-consistency=false;strong-consistency-allow-expunge=false;tomb-raider-eligible-age=86400;tomb-raider-period=86400;transaction-pending-limit=20;truncate-threads=4;write-commit-level-override=off;xdr-bin-tombstone-ttl=86400;xdr-tomb-raider-period=120;xdr-tomb-raider-threads=1;geo2dsphere-within.strict=true;geo2dsphere-within.min-level=1;geo2dsphere-within.max-level=20;geo2dsphere-within.max-cells=12;geo2dsphere-within.level-mod=1;geo2dsphere-within.earth-radius-meters=6371000;index-type=shmem;sindex-type=shmem;storage-engine=memory +namespace-stats:bar_device:ns_cluster_size=1;effective_replication_factor=1;objects=0;tombstones=0;xdr_tombstones=0;xdr_bin_cemeteries=0;master_objects=0;master_tombstones=0;prole_objects=0;prole_tombstones=0;non_replica_objects=0;non_replica_tombstones=0;unreplicated_records=0;dead_partitions=0;unavailable_partitions=0;clock_skew_stop_writes=false;stop_writes=false;hwm_breached=false;current_time=427402124;non_expirable_objects=0;expired_objects=0;evicted_objects=0;evict_ttl=0;evict_void_time=0;smd_evict_void_time=0;nsup_cycle_duration=0;nsup_cycle_deleted_pct=0.00;truncate_lut=0;truncating=false;sindex_gc_cleaned=0;memory_used_bytes=0;memory_used_data_bytes=0;memory_used_index_bytes=0;memory_used_set_index_bytes=0;memory_used_sindex_bytes=0;memory_free_pct=100;xmem_id=2;available_bin_names=65535;device_total_bytes=17179869184;device_used_bytes=0;device_free_pct=100;device_available_pct=99;storage-engine.file[0].used_bytes=0;storage-engine.file[0].free_wblocks=16376;storage-engine.file[0].write_q=0;storage-engine.file[0].writes=0;storage-engine.file[0].defrag_q=0;storage-engine.file[0].defrag_reads=0;storage-engine.file[0].defrag_writes=0;storage-engine.file[0].shadow_write_q=0;storage-engine.file[0].age=-1;record_proto_uncompressed_pct=0.000;record_proto_compression_ratio=1.000;query_proto_uncompressed_pct=0.000;query_proto_compression_ratio=1.000;pending_quiesce=false;effective_is_quiesced=false;nodes_quiesced=0;effective_prefer_uniform_balance=true;migrate_tx_partitions_imbalance=0;migrate_tx_instances=0;migrate_rx_instances=0;migrate_tx_partitions_active=0;migrate_rx_partitions_active=0;migrate_tx_partitions_initial=0;migrate_tx_partitions_remaining=0;migrate_tx_partitions_lead_remaining=0;migrate_rx_partitions_initial=0;migrate_rx_partitions_remaining=0;migrate_records_skipped=0;migrate_records_transmitted=0;migrate_record_retransmits=0;migrate_record_receives=0;migrate_signals_active=0;migrate_signals_remaining=0;appeals_tx_active=0;appeals_rx_active=0;appeals_tx_remaining=0;appeals_records_exonerated=0;client_tsvc_error=0;client_tsvc_timeout=0;client_proxy_complete=0;client_proxy_error=0;client_proxy_timeout=0;client_read_success=0;client_read_error=0;client_read_timeout=0;client_read_not_found=0;client_read_filtered_out=0;client_write_success=0;client_write_error=0;client_write_timeout=0;client_write_filtered_out=0;xdr_client_write_success=0;xdr_client_write_error=0;xdr_client_write_timeout=0;client_delete_success=0;client_delete_error=0;client_delete_timeout=0;client_delete_not_found=0;client_delete_filtered_out=0;xdr_client_delete_success=0;xdr_client_delete_error=0;xdr_client_delete_timeout=0;xdr_client_delete_not_found=0;client_udf_complete=0;client_udf_error=0;client_udf_timeout=0;client_udf_filtered_out=0;client_lang_read_success=0;client_lang_write_success=0;client_lang_delete_success=0;client_lang_error=0;from_proxy_tsvc_error=0;from_proxy_tsvc_timeout=0;from_proxy_read_success=0;from_proxy_read_error=0;from_proxy_read_timeout=0;from_proxy_read_not_found=0;from_proxy_read_filtered_out=0;from_proxy_write_success=0;from_proxy_write_error=0;from_proxy_write_timeout=0;from_proxy_write_filtered_out=0;xdr_from_proxy_write_success=0;xdr_from_proxy_write_error=0;xdr_from_proxy_write_timeout=0;from_proxy_delete_success=0;from_proxy_delete_error=0;from_proxy_delete_timeout=0;from_proxy_delete_not_found=0;from_proxy_delete_filtered_out=0;xdr_from_proxy_delete_success=0;xdr_from_proxy_delete_error=0;xdr_from_proxy_delete_timeout=0;xdr_from_proxy_delete_not_found=0;from_proxy_udf_complete=0;from_proxy_udf_error=0;from_proxy_udf_timeout=0;from_proxy_udf_filtered_out=0;from_proxy_lang_read_success=0;from_proxy_lang_write_success=0;from_proxy_lang_delete_success=0;from_proxy_lang_error=0;batch_sub_tsvc_error=0;batch_sub_tsvc_timeout=0;batch_sub_proxy_complete=0;batch_sub_proxy_error=0;batch_sub_proxy_timeout=0;batch_sub_read_success=0;batch_sub_read_error=0;batch_sub_read_timeout=0;batch_sub_read_not_found=0;batch_sub_read_filtered_out=0;batch_sub_write_success=0;batch_sub_write_error=0;batch_sub_write_timeout=0;batch_sub_write_filtered_out=0;batch_sub_delete_success=0;batch_sub_delete_error=0;batch_sub_delete_timeout=0;batch_sub_delete_not_found=0;batch_sub_delete_filtered_out=0;batch_sub_udf_complete=0;batch_sub_udf_error=0;batch_sub_udf_timeout=0;batch_sub_udf_filtered_out=0;batch_sub_lang_read_success=0;batch_sub_lang_write_success=0;batch_sub_lang_delete_success=0;batch_sub_lang_error=0;from_proxy_batch_sub_tsvc_error=0;from_proxy_batch_sub_tsvc_timeout=0;from_proxy_batch_sub_read_success=0;from_proxy_batch_sub_read_error=0;from_proxy_batch_sub_read_timeout=0;from_proxy_batch_sub_read_not_found=0;from_proxy_batch_sub_read_filtered_out=0;from_proxy_batch_sub_write_success=0;from_proxy_batch_sub_write_error=0;from_proxy_batch_sub_write_timeout=0;from_proxy_batch_sub_write_filtered_out=0;from_proxy_batch_sub_delete_success=0;from_proxy_batch_sub_delete_error=0;from_proxy_batch_sub_delete_timeout=0;from_proxy_batch_sub_delete_not_found=0;from_proxy_batch_sub_delete_filtered_out=0;from_proxy_batch_sub_udf_complete=0;from_proxy_batch_sub_udf_error=0;from_proxy_batch_sub_udf_timeout=0;from_proxy_batch_sub_udf_filtered_out=0;from_proxy_batch_sub_lang_read_success=0;from_proxy_batch_sub_lang_write_success=0;from_proxy_batch_sub_lang_delete_success=0;from_proxy_batch_sub_lang_error=0;udf_sub_tsvc_error=0;udf_sub_tsvc_timeout=0;udf_sub_udf_complete=0;udf_sub_udf_error=0;udf_sub_udf_timeout=0;udf_sub_udf_filtered_out=0;udf_sub_lang_read_success=0;udf_sub_lang_write_success=0;udf_sub_lang_delete_success=0;udf_sub_lang_error=0;ops_sub_tsvc_error=0;ops_sub_tsvc_timeout=0;ops_sub_write_success=0;ops_sub_write_error=0;ops_sub_write_timeout=0;ops_sub_write_filtered_out=0;dup_res_ask=0;dup_res_respond_read=0;dup_res_respond_no_read=0;retransmit_all_read_dup_res=0;retransmit_all_write_dup_res=0;retransmit_all_write_repl_write=0;retransmit_all_delete_dup_res=0;retransmit_all_delete_repl_write=0;retransmit_all_udf_dup_res=0;retransmit_all_udf_repl_write=0;retransmit_all_batch_sub_dup_res=0;retransmit_udf_sub_dup_res=0;retransmit_udf_sub_repl_write=0;retransmit_ops_sub_dup_res=0;retransmit_ops_sub_repl_write=0;pi_query_short_basic_complete=0;pi_query_short_basic_error=0;pi_query_short_basic_timeout=0;pi_query_long_basic_complete=0;pi_query_long_basic_error=0;pi_query_long_basic_abort=0;pi_query_aggr_complete=0;pi_query_aggr_error=0;pi_query_aggr_abort=0;pi_query_udf_bg_complete=0;pi_query_udf_bg_error=0;pi_query_udf_bg_abort=0;pi_query_ops_bg_complete=0;pi_query_ops_bg_error=0;pi_query_ops_bg_abort=0;si_query_short_basic_complete=0;si_query_short_basic_error=0;si_query_short_basic_timeout=0;si_query_long_basic_complete=0;si_query_long_basic_error=0;si_query_long_basic_abort=0;si_query_aggr_complete=0;si_query_aggr_error=0;si_query_aggr_abort=0;si_query_udf_bg_complete=0;si_query_udf_bg_error=0;si_query_udf_bg_abort=0;si_query_ops_bg_complete=0;si_query_ops_bg_error=0;si_query_ops_bg_abort=0;geo_region_query_reqs=0;geo_region_query_cells=0;geo_region_query_points=0;geo_region_query_falsepos=0;re_repl_tsvc_error=0;re_repl_tsvc_timeout=0;re_repl_success=0;re_repl_error=0;re_repl_timeout=0;fail_xdr_forbidden=0;fail_key_busy=0;fail_generation=0;fail_record_too_big=0;fail_client_lost_conflict=0;fail_xdr_lost_conflict=0;deleted_last_bin=0;allow-ttl-without-nsup=false;background-query-max-rps=10000;conflict-resolution-policy=generation;conflict-resolve-writes=false;default-ttl=0;disable-cold-start-eviction=false;disable-write-dup-res=false;disallow-expunge=false;disallow-null-setname=false;enable-benchmarks-batch-sub=false;enable-benchmarks-ops-sub=false;enable-benchmarks-read=false;enable-benchmarks-udf=false;enable-benchmarks-udf-sub=false;enable-benchmarks-write=false;enable-hist-proxy=false;evict-hist-buckets=10000;evict-tenths-pct=5;force-long-queries=false;high-water-disk-pct=0;high-water-memory-pct=0;ignore-migrate-fill-delay=false;index-stage-size=1073741824;inline-short-queries=false;max-record-size=0;memory-size=4294967296;migrate-order=5;migrate-retransmit-ms=5000;migrate-sleep=1;nsup-hist-period=3600;nsup-period=0;nsup-threads=1;partition-tree-sprigs=256;prefer-uniform-balance=true;rack-id=0;read-consistency-level-override=off;reject-non-xdr-writes=false;reject-xdr-writes=false;replication-factor=2;sindex-stage-size=1073741824;single-query-threads=4;stop-writes-pct=90;stop-writes-sys-memory-pct=90;strong-consistency=false;strong-consistency-allow-expunge=false;tomb-raider-eligible-age=86400;tomb-raider-period=86400;transaction-pending-limit=20;truncate-threads=4;write-commit-level-override=off;xdr-bin-tombstone-ttl=86400;xdr-tomb-raider-period=120;xdr-tomb-raider-threads=1;geo2dsphere-within.strict=true;geo2dsphere-within.min-level=1;geo2dsphere-within.max-level=20;geo2dsphere-within.max-cells=12;geo2dsphere-within.level-mod=1;geo2dsphere-within.earth-radius-meters=6371000;index-type=shmem;sindex-type=shmem;storage-engine=device;storage-engine.file[0]=/root/data/bar_device.dat;storage-engine.cache-replica-writes=false;storage-engine.cold-start-empty=false;storage-engine.commit-to-device=false;storage-engine.commit-min-size=0;storage-engine.compression=none;storage-engine.compression-acceleration=0;storage-engine.compression-level=0;storage-engine.data-in-memory=true;storage-engine.defrag-lwm-pct=50;storage-engine.defrag-queue-min=0;storage-engine.defrag-sleep=1000;storage-engine.defrag-startup-minimum=0;storage-engine.direct-files=false;storage-engine.disable-odsync=false;storage-engine.enable-benchmarks-storage=false;storage-engine.encryption-key-file=null;storage-engine.encryption-old-key-file=null;storage-engine.filesize=17179869184;storage-engine.flush-max-ms=1000;storage-engine.max-used-pct=70;storage-engine.max-write-cache=67108864;storage-engine.min-avail-pct=5;storage-engine.post-write-queue=0;storage-engine.read-page-cache=false;storage-engine.serialize-tomb-raider=false;storage-engine.sindex-startup-device-scan=false;storage-engine.tomb-raider-sleep=1000;storage-engine.write-block-size=1048576 +namespace-stats:materials:ns_cluster_size=1;effective_replication_factor=1;objects=0;tombstones=0;xdr_tombstones=0;xdr_bin_cemeteries=0;master_objects=0;master_tombstones=0;prole_objects=0;prole_tombstones=0;non_replica_objects=0;non_replica_tombstones=0;unreplicated_records=0;dead_partitions=0;unavailable_partitions=0;clock_skew_stop_writes=false;stop_writes=false;hwm_breached=false;current_time=427402124;non_expirable_objects=0;expired_objects=0;evicted_objects=0;evict_ttl=0;evict_void_time=0;smd_evict_void_time=0;nsup_cycle_duration=0;nsup_cycle_deleted_pct=0.00;truncate_lut=0;truncating=false;sindex_gc_cleaned=0;memory_used_bytes=0;memory_used_data_bytes=0;memory_used_index_bytes=0;memory_used_set_index_bytes=0;memory_used_sindex_bytes=0;memory_free_pct=100;xmem_id=3;available_bin_names=65535;record_proto_uncompressed_pct=0.000;record_proto_compression_ratio=1.000;query_proto_uncompressed_pct=0.000;query_proto_compression_ratio=1.000;pending_quiesce=false;effective_is_quiesced=false;nodes_quiesced=0;effective_prefer_uniform_balance=true;migrate_tx_partitions_imbalance=0;migrate_tx_instances=0;migrate_rx_instances=0;migrate_tx_partitions_active=0;migrate_rx_partitions_active=0;migrate_tx_partitions_initial=0;migrate_tx_partitions_remaining=0;migrate_tx_partitions_lead_remaining=0;migrate_rx_partitions_initial=0;migrate_rx_partitions_remaining=0;migrate_records_skipped=0;migrate_records_transmitted=0;migrate_record_retransmits=0;migrate_record_receives=0;migrate_signals_active=0;migrate_signals_remaining=0;appeals_tx_active=0;appeals_rx_active=0;appeals_tx_remaining=0;appeals_records_exonerated=0;client_tsvc_error=0;client_tsvc_timeout=0;client_proxy_complete=0;client_proxy_error=0;client_proxy_timeout=0;client_read_success=0;client_read_error=0;client_read_timeout=0;client_read_not_found=0;client_read_filtered_out=0;client_write_success=0;client_write_error=0;client_write_timeout=0;client_write_filtered_out=0;xdr_client_write_success=0;xdr_client_write_error=0;xdr_client_write_timeout=0;client_delete_success=0;client_delete_error=0;client_delete_timeout=0;client_delete_not_found=0;client_delete_filtered_out=0;xdr_client_delete_success=0;xdr_client_delete_error=0;xdr_client_delete_timeout=0;xdr_client_delete_not_found=0;client_udf_complete=0;client_udf_error=0;client_udf_timeout=0;client_udf_filtered_out=0;client_lang_read_success=0;client_lang_write_success=0;client_lang_delete_success=0;client_lang_error=0;from_proxy_tsvc_error=0;from_proxy_tsvc_timeout=0;from_proxy_read_success=0;from_proxy_read_error=0;from_proxy_read_timeout=0;from_proxy_read_not_found=0;from_proxy_read_filtered_out=0;from_proxy_write_success=0;from_proxy_write_error=0;from_proxy_write_timeout=0;from_proxy_write_filtered_out=0;xdr_from_proxy_write_success=0;xdr_from_proxy_write_error=0;xdr_from_proxy_write_timeout=0;from_proxy_delete_success=0;from_proxy_delete_error=0;from_proxy_delete_timeout=0;from_proxy_delete_not_found=0;from_proxy_delete_filtered_out=0;xdr_from_proxy_delete_success=0;xdr_from_proxy_delete_error=0;xdr_from_proxy_delete_timeout=0;xdr_from_proxy_delete_not_found=0;from_proxy_udf_complete=0;from_proxy_udf_error=0;from_proxy_udf_timeout=0;from_proxy_udf_filtered_out=0;from_proxy_lang_read_success=0;from_proxy_lang_write_success=0;from_proxy_lang_delete_success=0;from_proxy_lang_error=0;batch_sub_tsvc_error=0;batch_sub_tsvc_timeout=0;batch_sub_proxy_complete=0;batch_sub_proxy_error=0;batch_sub_proxy_timeout=0;batch_sub_read_success=0;batch_sub_read_error=0;batch_sub_read_timeout=0;batch_sub_read_not_found=0;batch_sub_read_filtered_out=0;batch_sub_write_success=0;batch_sub_write_error=0;batch_sub_write_timeout=0;batch_sub_write_filtered_out=0;batch_sub_delete_success=0;batch_sub_delete_error=0;batch_sub_delete_timeout=0;batch_sub_delete_not_found=0;batch_sub_delete_filtered_out=0;batch_sub_udf_complete=0;batch_sub_udf_error=0;batch_sub_udf_timeout=0;batch_sub_udf_filtered_out=0;batch_sub_lang_read_success=0;batch_sub_lang_write_success=0;batch_sub_lang_delete_success=0;batch_sub_lang_error=0;from_proxy_batch_sub_tsvc_error=0;from_proxy_batch_sub_tsvc_timeout=0;from_proxy_batch_sub_read_success=0;from_proxy_batch_sub_read_error=0;from_proxy_batch_sub_read_timeout=0;from_proxy_batch_sub_read_not_found=0;from_proxy_batch_sub_read_filtered_out=0;from_proxy_batch_sub_write_success=0;from_proxy_batch_sub_write_error=0;from_proxy_batch_sub_write_timeout=0;from_proxy_batch_sub_write_filtered_out=0;from_proxy_batch_sub_delete_success=0;from_proxy_batch_sub_delete_error=0;from_proxy_batch_sub_delete_timeout=0;from_proxy_batch_sub_delete_not_found=0;from_proxy_batch_sub_delete_filtered_out=0;from_proxy_batch_sub_udf_complete=0;from_proxy_batch_sub_udf_error=0;from_proxy_batch_sub_udf_timeout=0;from_proxy_batch_sub_udf_filtered_out=0;from_proxy_batch_sub_lang_read_success=0;from_proxy_batch_sub_lang_write_success=0;from_proxy_batch_sub_lang_delete_success=0;from_proxy_batch_sub_lang_error=0;udf_sub_tsvc_error=0;udf_sub_tsvc_timeout=0;udf_sub_udf_complete=0;udf_sub_udf_error=0;udf_sub_udf_timeout=0;udf_sub_udf_filtered_out=0;udf_sub_lang_read_success=0;udf_sub_lang_write_success=0;udf_sub_lang_delete_success=0;udf_sub_lang_error=0;ops_sub_tsvc_error=0;ops_sub_tsvc_timeout=0;ops_sub_write_success=0;ops_sub_write_error=0;ops_sub_write_timeout=0;ops_sub_write_filtered_out=0;dup_res_ask=0;dup_res_respond_read=0;dup_res_respond_no_read=0;retransmit_all_read_dup_res=0;retransmit_all_write_dup_res=0;retransmit_all_write_repl_write=0;retransmit_all_delete_dup_res=0;retransmit_all_delete_repl_write=0;retransmit_all_udf_dup_res=0;retransmit_all_udf_repl_write=0;retransmit_all_batch_sub_dup_res=0;retransmit_udf_sub_dup_res=0;retransmit_udf_sub_repl_write=0;retransmit_ops_sub_dup_res=0;retransmit_ops_sub_repl_write=0;pi_query_short_basic_complete=0;pi_query_short_basic_error=0;pi_query_short_basic_timeout=0;pi_query_long_basic_complete=0;pi_query_long_basic_error=0;pi_query_long_basic_abort=0;pi_query_aggr_complete=0;pi_query_aggr_error=0;pi_query_aggr_abort=0;pi_query_udf_bg_complete=0;pi_query_udf_bg_error=0;pi_query_udf_bg_abort=0;pi_query_ops_bg_complete=0;pi_query_ops_bg_error=0;pi_query_ops_bg_abort=0;si_query_short_basic_complete=0;si_query_short_basic_error=0;si_query_short_basic_timeout=0;si_query_long_basic_complete=0;si_query_long_basic_error=0;si_query_long_basic_abort=0;si_query_aggr_complete=0;si_query_aggr_error=0;si_query_aggr_abort=0;si_query_udf_bg_complete=0;si_query_udf_bg_error=0;si_query_udf_bg_abort=0;si_query_ops_bg_complete=0;si_query_ops_bg_error=0;si_query_ops_bg_abort=0;geo_region_query_reqs=0;geo_region_query_cells=0;geo_region_query_points=0;geo_region_query_falsepos=0;re_repl_tsvc_error=0;re_repl_tsvc_timeout=0;re_repl_success=0;re_repl_error=0;re_repl_timeout=0;fail_xdr_forbidden=0;fail_key_busy=0;fail_generation=0;fail_record_too_big=0;fail_client_lost_conflict=0;fail_xdr_lost_conflict=0;deleted_last_bin=0;allow-ttl-without-nsup=false;background-query-max-rps=10000;conflict-resolution-policy=generation;conflict-resolve-writes=false;default-ttl=0;disable-cold-start-eviction=false;disable-write-dup-res=false;disallow-expunge=false;disallow-null-setname=false;enable-benchmarks-batch-sub=false;enable-benchmarks-ops-sub=false;enable-benchmarks-read=false;enable-benchmarks-udf=false;enable-benchmarks-udf-sub=false;enable-benchmarks-write=false;enable-hist-proxy=false;evict-hist-buckets=10000;evict-tenths-pct=5;force-long-queries=false;high-water-disk-pct=0;high-water-memory-pct=0;ignore-migrate-fill-delay=false;index-stage-size=1073741824;inline-short-queries=false;max-record-size=0;memory-size=4294967296;migrate-order=5;migrate-retransmit-ms=5000;migrate-sleep=1;nsup-hist-period=3600;nsup-period=0;nsup-threads=1;partition-tree-sprigs=256;prefer-uniform-balance=true;rack-id=0;read-consistency-level-override=off;reject-non-xdr-writes=false;reject-xdr-writes=false;replication-factor=2;sindex-stage-size=1073741824;single-query-threads=4;stop-writes-pct=90;stop-writes-sys-memory-pct=90;strong-consistency=false;strong-consistency-allow-expunge=false;tomb-raider-eligible-age=86400;tomb-raider-period=86400;transaction-pending-limit=20;truncate-threads=4;write-commit-level-override=off;xdr-bin-tombstone-ttl=86400;xdr-tomb-raider-period=120;xdr-tomb-raider-threads=1;geo2dsphere-within.strict=true;geo2dsphere-within.min-level=1;geo2dsphere-within.max-level=20;geo2dsphere-within.max-cells=12;geo2dsphere-within.level-mod=1;geo2dsphere-within.earth-radius-meters=6371000;index-type=shmem;sindex-type=shmem;storage-engine=memory +namespace-stats:ns_test_on_flash:ns_cluster_size=1;effective_replication_factor=1;objects=6273598;tombstones=0;xdr_tombstones=0;xdr_bin_cemeteries=0;master_objects=6273603;master_tombstones=0;prole_objects=0;prole_tombstones=0;non_replica_objects=0;non_replica_tombstones=0;unreplicated_records=0;dead_partitions=0;unavailable_partitions=0;clock_skew_stop_writes=false;stop_writes=false;hwm_breached=false;current_time=427402124;non_expirable_objects=0;expired_objects=0;evicted_objects=0;evict_ttl=0;evict_void_time=0;smd_evict_void_time=0;nsup_cycle_duration=0;nsup_cycle_deleted_pct=0.00;truncate_lut=0;truncating=false;sindex_gc_cleaned=0;memory_used_bytes=431593285;memory_used_data_bytes=431593285;memory_used_index_bytes=0;memory_used_set_index_bytes=0;memory_used_sindex_bytes=0;memory_free_pct=89;xmem_id=4;available_bin_names=65532;index_flash_used_bytes=401511232;index_flash_used_pct=9;index_flash_alloc_bytes=4284223488;index_flash_alloc_pct=99;index-type.mount[0].age=-1;index-type.mount[1].age=-1;record_proto_uncompressed_pct=0.000;record_proto_compression_ratio=1.000;query_proto_uncompressed_pct=0.000;query_proto_compression_ratio=1.000;pending_quiesce=false;effective_is_quiesced=false;nodes_quiesced=0;effective_prefer_uniform_balance=true;migrate_tx_partitions_imbalance=0;migrate_tx_instances=0;migrate_rx_instances=0;migrate_tx_partitions_active=0;migrate_rx_partitions_active=0;migrate_tx_partitions_initial=0;migrate_tx_partitions_remaining=0;migrate_tx_partitions_lead_remaining=0;migrate_rx_partitions_initial=0;migrate_rx_partitions_remaining=0;migrate_records_skipped=0;migrate_records_transmitted=0;migrate_record_retransmits=0;migrate_record_receives=0;migrate_signals_active=0;migrate_signals_remaining=0;appeals_tx_active=0;appeals_rx_active=0;appeals_tx_remaining=0;appeals_records_exonerated=0;client_tsvc_error=0;client_tsvc_timeout=0;client_proxy_complete=0;client_proxy_error=0;client_proxy_timeout=0;client_read_success=0;client_read_error=0;client_read_timeout=0;client_read_not_found=0;client_read_filtered_out=0;client_write_success=8706341;client_write_error=0;client_write_timeout=0;client_write_filtered_out=0;xdr_client_write_success=0;xdr_client_write_error=0;xdr_client_write_timeout=0;client_delete_success=0;client_delete_error=0;client_delete_timeout=0;client_delete_not_found=0;client_delete_filtered_out=0;xdr_client_delete_success=0;xdr_client_delete_error=0;xdr_client_delete_timeout=0;xdr_client_delete_not_found=0;client_udf_complete=0;client_udf_error=0;client_udf_timeout=0;client_udf_filtered_out=0;client_lang_read_success=0;client_lang_write_success=0;client_lang_delete_success=0;client_lang_error=0;from_proxy_tsvc_error=0;from_proxy_tsvc_timeout=0;from_proxy_read_success=0;from_proxy_read_error=0;from_proxy_read_timeout=0;from_proxy_read_not_found=0;from_proxy_read_filtered_out=0;from_proxy_write_success=0;from_proxy_write_error=0;from_proxy_write_timeout=0;from_proxy_write_filtered_out=0;xdr_from_proxy_write_success=0;xdr_from_proxy_write_error=0;xdr_from_proxy_write_timeout=0;from_proxy_delete_success=0;from_proxy_delete_error=0;from_proxy_delete_timeout=0;from_proxy_delete_not_found=0;from_proxy_delete_filtered_out=0;xdr_from_proxy_delete_success=0;xdr_from_proxy_delete_error=0;xdr_from_proxy_delete_timeout=0;xdr_from_proxy_delete_not_found=0;from_proxy_udf_complete=0;from_proxy_udf_error=0;from_proxy_udf_timeout=0;from_proxy_udf_filtered_out=0;from_proxy_lang_read_success=0;from_proxy_lang_write_success=0;from_proxy_lang_delete_success=0;from_proxy_lang_error=0;batch_sub_tsvc_error=0;batch_sub_tsvc_timeout=0;batch_sub_proxy_complete=0;batch_sub_proxy_error=0;batch_sub_proxy_timeout=0;batch_sub_read_success=0;batch_sub_read_error=0;batch_sub_read_timeout=0;batch_sub_read_not_found=0;batch_sub_read_filtered_out=0;batch_sub_write_success=0;batch_sub_write_error=0;batch_sub_write_timeout=0;batch_sub_write_filtered_out=0;batch_sub_delete_success=0;batch_sub_delete_error=0;batch_sub_delete_timeout=0;batch_sub_delete_not_found=0;batch_sub_delete_filtered_out=0;batch_sub_udf_complete=0;batch_sub_udf_error=0;batch_sub_udf_timeout=0;batch_sub_udf_filtered_out=0;batch_sub_lang_read_success=0;batch_sub_lang_write_success=0;batch_sub_lang_delete_success=0;batch_sub_lang_error=0;from_proxy_batch_sub_tsvc_error=0;from_proxy_batch_sub_tsvc_timeout=0;from_proxy_batch_sub_read_success=0;from_proxy_batch_sub_read_error=0;from_proxy_batch_sub_read_timeout=0;from_proxy_batch_sub_read_not_found=0;from_proxy_batch_sub_read_filtered_out=0;from_proxy_batch_sub_write_success=0;from_proxy_batch_sub_write_error=0;from_proxy_batch_sub_write_timeout=0;from_proxy_batch_sub_write_filtered_out=0;from_proxy_batch_sub_delete_success=0;from_proxy_batch_sub_delete_error=0;from_proxy_batch_sub_delete_timeout=0;from_proxy_batch_sub_delete_not_found=0;from_proxy_batch_sub_delete_filtered_out=0;from_proxy_batch_sub_udf_complete=0;from_proxy_batch_sub_udf_error=0;from_proxy_batch_sub_udf_timeout=0;from_proxy_batch_sub_udf_filtered_out=0;from_proxy_batch_sub_lang_read_success=0;from_proxy_batch_sub_lang_write_success=0;from_proxy_batch_sub_lang_delete_success=0;from_proxy_batch_sub_lang_error=0;udf_sub_tsvc_error=0;udf_sub_tsvc_timeout=0;udf_sub_udf_complete=0;udf_sub_udf_error=0;udf_sub_udf_timeout=0;udf_sub_udf_filtered_out=0;udf_sub_lang_read_success=0;udf_sub_lang_write_success=0;udf_sub_lang_delete_success=0;udf_sub_lang_error=0;ops_sub_tsvc_error=0;ops_sub_tsvc_timeout=0;ops_sub_write_success=0;ops_sub_write_error=0;ops_sub_write_timeout=0;ops_sub_write_filtered_out=0;dup_res_ask=0;dup_res_respond_read=0;dup_res_respond_no_read=0;retransmit_all_read_dup_res=0;retransmit_all_write_dup_res=0;retransmit_all_write_repl_write=0;retransmit_all_delete_dup_res=0;retransmit_all_delete_repl_write=0;retransmit_all_udf_dup_res=0;retransmit_all_udf_repl_write=0;retransmit_all_batch_sub_dup_res=0;retransmit_udf_sub_dup_res=0;retransmit_udf_sub_repl_write=0;retransmit_ops_sub_dup_res=0;retransmit_ops_sub_repl_write=0;pi_query_short_basic_complete=0;pi_query_short_basic_error=0;pi_query_short_basic_timeout=0;pi_query_long_basic_complete=0;pi_query_long_basic_error=0;pi_query_long_basic_abort=0;pi_query_aggr_complete=0;pi_query_aggr_error=0;pi_query_aggr_abort=0;pi_query_udf_bg_complete=0;pi_query_udf_bg_error=0;pi_query_udf_bg_abort=0;pi_query_ops_bg_complete=0;pi_query_ops_bg_error=0;pi_query_ops_bg_abort=0;si_query_short_basic_complete=0;si_query_short_basic_error=0;si_query_short_basic_timeout=0;si_query_long_basic_complete=0;si_query_long_basic_error=0;si_query_long_basic_abort=0;si_query_aggr_complete=0;si_query_aggr_error=0;si_query_aggr_abort=0;si_query_udf_bg_complete=0;si_query_udf_bg_error=0;si_query_udf_bg_abort=0;si_query_ops_bg_complete=0;si_query_ops_bg_error=0;si_query_ops_bg_abort=0;geo_region_query_reqs=0;geo_region_query_cells=0;geo_region_query_points=0;geo_region_query_falsepos=0;re_repl_tsvc_error=0;re_repl_tsvc_timeout=0;re_repl_success=0;re_repl_error=0;re_repl_timeout=0;fail_xdr_forbidden=0;fail_key_busy=0;fail_generation=0;fail_record_too_big=0;fail_client_lost_conflict=0;fail_xdr_lost_conflict=0;deleted_last_bin=0;allow-ttl-without-nsup=false;background-query-max-rps=10000;conflict-resolution-policy=generation;conflict-resolve-writes=false;default-ttl=0;disable-cold-start-eviction=false;disable-write-dup-res=false;disallow-expunge=false;disallow-null-setname=false;enable-benchmarks-batch-sub=false;enable-benchmarks-ops-sub=false;enable-benchmarks-read=false;enable-benchmarks-udf=false;enable-benchmarks-udf-sub=false;enable-benchmarks-write=false;enable-hist-proxy=false;evict-hist-buckets=10000;evict-tenths-pct=5;force-long-queries=false;high-water-disk-pct=0;high-water-memory-pct=20;ignore-migrate-fill-delay=false;index-stage-size=1073741824;inline-short-queries=false;max-record-size=0;memory-size=4294967296;migrate-order=5;migrate-retransmit-ms=5000;migrate-sleep=1;nsup-hist-period=3600;nsup-period=0;nsup-threads=1;partition-tree-sprigs=256;prefer-uniform-balance=true;rack-id=0;read-consistency-level-override=off;reject-non-xdr-writes=false;reject-xdr-writes=false;replication-factor=2;sindex-stage-size=1073741824;single-query-threads=4;stop-writes-pct=90;stop-writes-sys-memory-pct=90;strong-consistency=false;strong-consistency-allow-expunge=false;tomb-raider-eligible-age=86400;tomb-raider-period=86400;transaction-pending-limit=20;truncate-threads=4;write-commit-level-override=off;xdr-bin-tombstone-ttl=86400;xdr-tomb-raider-period=120;xdr-tomb-raider-threads=1;geo2dsphere-within.strict=true;geo2dsphere-within.min-level=1;geo2dsphere-within.max-level=20;geo2dsphere-within.max-cells=12;geo2dsphere-within.level-mod=1;geo2dsphere-within.earth-radius-meters=6371000;index-type=flash;index-type.mount[0]=/root/data/flash_disk_mount;index-type.mount[1]=/root/data/flash_mount_drive_2;index-type.mounts-high-water-pct=20;index-type.mounts-size-limit=4294967296;sindex-type=shmem;storage-engine=memory +namespace-stats:test_on_shmem:ns_cluster_size=1;effective_replication_factor=1;objects=330161;tombstones=0;xdr_tombstones=0;xdr_bin_cemeteries=0;master_objects=330161;master_tombstones=0;prole_objects=0;prole_tombstones=0;non_replica_objects=0;non_replica_tombstones=0;unreplicated_records=0;dead_partitions=0;unavailable_partitions=0;clock_skew_stop_writes=false;stop_writes=false;hwm_breached=false;current_time=427402124;non_expirable_objects=0;expired_objects=0;evicted_objects=0;evict_ttl=0;evict_void_time=0;smd_evict_void_time=0;nsup_cycle_duration=0;nsup_cycle_deleted_pct=0.00;truncate_lut=0;truncating=false;sindex_gc_cleaned=0;memory_used_bytes=43805428;memory_used_data_bytes=22675124;memory_used_index_bytes=21130304;memory_used_set_index_bytes=0;memory_used_sindex_bytes=0;memory_free_pct=98;xmem_id=5;available_bin_names=65532;record_proto_uncompressed_pct=0.000;record_proto_compression_ratio=1.000;query_proto_uncompressed_pct=0.000;query_proto_compression_ratio=1.000;pending_quiesce=false;effective_is_quiesced=false;nodes_quiesced=0;effective_prefer_uniform_balance=true;migrate_tx_partitions_imbalance=0;migrate_tx_instances=0;migrate_rx_instances=0;migrate_tx_partitions_active=0;migrate_rx_partitions_active=0;migrate_tx_partitions_initial=0;migrate_tx_partitions_remaining=0;migrate_tx_partitions_lead_remaining=0;migrate_rx_partitions_initial=0;migrate_rx_partitions_remaining=0;migrate_records_skipped=0;migrate_records_transmitted=0;migrate_record_retransmits=0;migrate_record_receives=0;migrate_signals_active=0;migrate_signals_remaining=0;appeals_tx_active=0;appeals_rx_active=0;appeals_tx_remaining=0;appeals_records_exonerated=0;client_tsvc_error=0;client_tsvc_timeout=0;client_proxy_complete=0;client_proxy_error=0;client_proxy_timeout=0;client_read_success=0;client_read_error=0;client_read_timeout=0;client_read_not_found=0;client_read_filtered_out=0;client_write_success=330161;client_write_error=0;client_write_timeout=0;client_write_filtered_out=0;xdr_client_write_success=0;xdr_client_write_error=0;xdr_client_write_timeout=0;client_delete_success=0;client_delete_error=0;client_delete_timeout=0;client_delete_not_found=0;client_delete_filtered_out=0;xdr_client_delete_success=0;xdr_client_delete_error=0;xdr_client_delete_timeout=0;xdr_client_delete_not_found=0;client_udf_complete=0;client_udf_error=0;client_udf_timeout=0;client_udf_filtered_out=0;client_lang_read_success=0;client_lang_write_success=0;client_lang_delete_success=0;client_lang_error=0;from_proxy_tsvc_error=0;from_proxy_tsvc_timeout=0;from_proxy_read_success=0;from_proxy_read_error=0;from_proxy_read_timeout=0;from_proxy_read_not_found=0;from_proxy_read_filtered_out=0;from_proxy_write_success=0;from_proxy_write_error=0;from_proxy_write_timeout=0;from_proxy_write_filtered_out=0;xdr_from_proxy_write_success=0;xdr_from_proxy_write_error=0;xdr_from_proxy_write_timeout=0;from_proxy_delete_success=0;from_proxy_delete_error=0;from_proxy_delete_timeout=0;from_proxy_delete_not_found=0;from_proxy_delete_filtered_out=0;xdr_from_proxy_delete_success=0;xdr_from_proxy_delete_error=0;xdr_from_proxy_delete_timeout=0;xdr_from_proxy_delete_not_found=0;from_proxy_udf_complete=0;from_proxy_udf_error=0;from_proxy_udf_timeout=0;from_proxy_udf_filtered_out=0;from_proxy_lang_read_success=0;from_proxy_lang_write_success=0;from_proxy_lang_delete_success=0;from_proxy_lang_error=0;batch_sub_tsvc_error=0;batch_sub_tsvc_timeout=0;batch_sub_proxy_complete=0;batch_sub_proxy_error=0;batch_sub_proxy_timeout=0;batch_sub_read_success=0;batch_sub_read_error=0;batch_sub_read_timeout=0;batch_sub_read_not_found=0;batch_sub_read_filtered_out=0;batch_sub_write_success=0;batch_sub_write_error=0;batch_sub_write_timeout=0;batch_sub_write_filtered_out=0;batch_sub_delete_success=0;batch_sub_delete_error=0;batch_sub_delete_timeout=0;batch_sub_delete_not_found=0;batch_sub_delete_filtered_out=0;batch_sub_udf_complete=0;batch_sub_udf_error=0;batch_sub_udf_timeout=0;batch_sub_udf_filtered_out=0;batch_sub_lang_read_success=0;batch_sub_lang_write_success=0;batch_sub_lang_delete_success=0;batch_sub_lang_error=0;from_proxy_batch_sub_tsvc_error=0;from_proxy_batch_sub_tsvc_timeout=0;from_proxy_batch_sub_read_success=0;from_proxy_batch_sub_read_error=0;from_proxy_batch_sub_read_timeout=0;from_proxy_batch_sub_read_not_found=0;from_proxy_batch_sub_read_filtered_out=0;from_proxy_batch_sub_write_success=0;from_proxy_batch_sub_write_error=0;from_proxy_batch_sub_write_timeout=0;from_proxy_batch_sub_write_filtered_out=0;from_proxy_batch_sub_delete_success=0;from_proxy_batch_sub_delete_error=0;from_proxy_batch_sub_delete_timeout=0;from_proxy_batch_sub_delete_not_found=0;from_proxy_batch_sub_delete_filtered_out=0;from_proxy_batch_sub_udf_complete=0;from_proxy_batch_sub_udf_error=0;from_proxy_batch_sub_udf_timeout=0;from_proxy_batch_sub_udf_filtered_out=0;from_proxy_batch_sub_lang_read_success=0;from_proxy_batch_sub_lang_write_success=0;from_proxy_batch_sub_lang_delete_success=0;from_proxy_batch_sub_lang_error=0;udf_sub_tsvc_error=0;udf_sub_tsvc_timeout=0;udf_sub_udf_complete=0;udf_sub_udf_error=0;udf_sub_udf_timeout=0;udf_sub_udf_filtered_out=0;udf_sub_lang_read_success=0;udf_sub_lang_write_success=0;udf_sub_lang_delete_success=0;udf_sub_lang_error=0;ops_sub_tsvc_error=0;ops_sub_tsvc_timeout=0;ops_sub_write_success=0;ops_sub_write_error=0;ops_sub_write_timeout=0;ops_sub_write_filtered_out=0;dup_res_ask=0;dup_res_respond_read=0;dup_res_respond_no_read=0;retransmit_all_read_dup_res=0;retransmit_all_write_dup_res=0;retransmit_all_write_repl_write=0;retransmit_all_delete_dup_res=0;retransmit_all_delete_repl_write=0;retransmit_all_udf_dup_res=0;retransmit_all_udf_repl_write=0;retransmit_all_batch_sub_dup_res=0;retransmit_udf_sub_dup_res=0;retransmit_udf_sub_repl_write=0;retransmit_ops_sub_dup_res=0;retransmit_ops_sub_repl_write=0;pi_query_short_basic_complete=0;pi_query_short_basic_error=0;pi_query_short_basic_timeout=0;pi_query_long_basic_complete=0;pi_query_long_basic_error=0;pi_query_long_basic_abort=0;pi_query_aggr_complete=0;pi_query_aggr_error=0;pi_query_aggr_abort=0;pi_query_udf_bg_complete=0;pi_query_udf_bg_error=0;pi_query_udf_bg_abort=0;pi_query_ops_bg_complete=0;pi_query_ops_bg_error=0;pi_query_ops_bg_abort=0;si_query_short_basic_complete=0;si_query_short_basic_error=0;si_query_short_basic_timeout=0;si_query_long_basic_complete=0;si_query_long_basic_error=0;si_query_long_basic_abort=0;si_query_aggr_complete=0;si_query_aggr_error=0;si_query_aggr_abort=0;si_query_udf_bg_complete=0;si_query_udf_bg_error=0;si_query_udf_bg_abort=0;si_query_ops_bg_complete=0;si_query_ops_bg_error=0;si_query_ops_bg_abort=0;geo_region_query_reqs=0;geo_region_query_cells=0;geo_region_query_points=0;geo_region_query_falsepos=0;re_repl_tsvc_error=0;re_repl_tsvc_timeout=0;re_repl_success=0;re_repl_error=0;re_repl_timeout=0;fail_xdr_forbidden=0;fail_key_busy=0;fail_generation=0;fail_record_too_big=0;fail_client_lost_conflict=0;fail_xdr_lost_conflict=0;deleted_last_bin=0;allow-ttl-without-nsup=false;background-query-max-rps=10000;conflict-resolution-policy=generation;conflict-resolve-writes=false;default-ttl=0;disable-cold-start-eviction=false;disable-write-dup-res=false;disallow-expunge=false;disallow-null-setname=false;enable-benchmarks-batch-sub=false;enable-benchmarks-ops-sub=false;enable-benchmarks-read=false;enable-benchmarks-udf=false;enable-benchmarks-udf-sub=false;enable-benchmarks-write=false;enable-hist-proxy=false;evict-hist-buckets=10000;evict-tenths-pct=5;force-long-queries=false;high-water-disk-pct=0;high-water-memory-pct=0;ignore-migrate-fill-delay=false;index-stage-size=1073741824;inline-short-queries=false;max-record-size=0;memory-size=4294967296;migrate-order=5;migrate-retransmit-ms=5000;migrate-sleep=1;nsup-hist-period=3600;nsup-period=0;nsup-threads=1;partition-tree-sprigs=256;prefer-uniform-balance=true;rack-id=0;read-consistency-level-override=off;reject-non-xdr-writes=false;reject-xdr-writes=false;replication-factor=2;sindex-stage-size=1073741824;single-query-threads=4;stop-writes-pct=90;stop-writes-sys-memory-pct=90;strong-consistency=false;strong-consistency-allow-expunge=false;tomb-raider-eligible-age=86400;tomb-raider-period=86400;transaction-pending-limit=20;truncate-threads=4;write-commit-level-override=off;xdr-bin-tombstone-ttl=86400;xdr-tomb-raider-period=120;xdr-tomb-raider-threads=1;geo2dsphere-within.strict=true;geo2dsphere-within.min-level=1;geo2dsphere-within.max-level=20;geo2dsphere-within.max-cells=12;geo2dsphere-within.level-mod=1;geo2dsphere-within.earth-radius-meters=6371000;index-type=shmem;sindex-type=shmem;storage-engine=memory +namespace-stats:bar_on_flash:ns_cluster_size=1;effective_replication_factor=1;objects=330169;tombstones=0;xdr_tombstones=0;xdr_bin_cemeteries=0;master_objects=330169;master_tombstones=0;prole_objects=0;prole_tombstones=0;non_replica_objects=0;non_replica_tombstones=0;unreplicated_records=0;dead_partitions=0;unavailable_partitions=0;clock_skew_stop_writes=false;stop_writes=false;hwm_breached=false;current_time=427402124;non_expirable_objects=0;expired_objects=0;evicted_objects=0;evict_ttl=0;evict_void_time=0;smd_evict_void_time=0;nsup_cycle_duration=0;nsup_cycle_deleted_pct=0.00;truncate_lut=0;truncating=false;sindex_gc_cleaned=0;memory_used_bytes=22675668;memory_used_data_bytes=22675668;memory_used_index_bytes=0;memory_used_set_index_bytes=0;memory_used_sindex_bytes=0;memory_free_pct=99;xmem_id=6;available_bin_names=65532;index_flash_used_bytes=21130816;index_flash_used_pct=0;index_flash_alloc_bytes=1160785920;index_flash_alloc_pct=27;index-type.mount[0].age=-1;sindex_flash_used_bytes=0;sindex_flash_used_pct=0;sindex-type.mount[0].age=-1;record_proto_uncompressed_pct=0.000;record_proto_compression_ratio=1.000;query_proto_uncompressed_pct=0.000;query_proto_compression_ratio=1.000;pending_quiesce=false;effective_is_quiesced=false;nodes_quiesced=0;effective_prefer_uniform_balance=true;migrate_tx_partitions_imbalance=0;migrate_tx_instances=0;migrate_rx_instances=0;migrate_tx_partitions_active=0;migrate_rx_partitions_active=0;migrate_tx_partitions_initial=0;migrate_tx_partitions_remaining=0;migrate_tx_partitions_lead_remaining=0;migrate_rx_partitions_initial=0;migrate_rx_partitions_remaining=0;migrate_records_skipped=0;migrate_records_transmitted=0;migrate_record_retransmits=0;migrate_record_receives=0;migrate_signals_active=0;migrate_signals_remaining=0;appeals_tx_active=0;appeals_rx_active=0;appeals_tx_remaining=0;appeals_records_exonerated=0;client_tsvc_error=0;client_tsvc_timeout=0;client_proxy_complete=0;client_proxy_error=0;client_proxy_timeout=0;client_read_success=0;client_read_error=0;client_read_timeout=0;client_read_not_found=0;client_read_filtered_out=0;client_write_success=330169;client_write_error=0;client_write_timeout=0;client_write_filtered_out=0;xdr_client_write_success=0;xdr_client_write_error=0;xdr_client_write_timeout=0;client_delete_success=0;client_delete_error=0;client_delete_timeout=0;client_delete_not_found=0;client_delete_filtered_out=0;xdr_client_delete_success=0;xdr_client_delete_error=0;xdr_client_delete_timeout=0;xdr_client_delete_not_found=0;client_udf_complete=0;client_udf_error=0;client_udf_timeout=0;client_udf_filtered_out=0;client_lang_read_success=0;client_lang_write_success=0;client_lang_delete_success=0;client_lang_error=0;from_proxy_tsvc_error=0;from_proxy_tsvc_timeout=0;from_proxy_read_success=0;from_proxy_read_error=0;from_proxy_read_timeout=0;from_proxy_read_not_found=0;from_proxy_read_filtered_out=0;from_proxy_write_success=0;from_proxy_write_error=0;from_proxy_write_timeout=0;from_proxy_write_filtered_out=0;xdr_from_proxy_write_success=0;xdr_from_proxy_write_error=0;xdr_from_proxy_write_timeout=0;from_proxy_delete_success=0;from_proxy_delete_error=0;from_proxy_delete_timeout=0;from_proxy_delete_not_found=0;from_proxy_delete_filtered_out=0;xdr_from_proxy_delete_success=0;xdr_from_proxy_delete_error=0;xdr_from_proxy_delete_timeout=0;xdr_from_proxy_delete_not_found=0;from_proxy_udf_complete=0;from_proxy_udf_error=0;from_proxy_udf_timeout=0;from_proxy_udf_filtered_out=0;from_proxy_lang_read_success=0;from_proxy_lang_write_success=0;from_proxy_lang_delete_success=0;from_proxy_lang_error=0;batch_sub_tsvc_error=0;batch_sub_tsvc_timeout=0;batch_sub_proxy_complete=0;batch_sub_proxy_error=0;batch_sub_proxy_timeout=0;batch_sub_read_success=0;batch_sub_read_error=0;batch_sub_read_timeout=0;batch_sub_read_not_found=0;batch_sub_read_filtered_out=0;batch_sub_write_success=0;batch_sub_write_error=0;batch_sub_write_timeout=0;batch_sub_write_filtered_out=0;batch_sub_delete_success=0;batch_sub_delete_error=0;batch_sub_delete_timeout=0;batch_sub_delete_not_found=0;batch_sub_delete_filtered_out=0;batch_sub_udf_complete=0;batch_sub_udf_error=0;batch_sub_udf_timeout=0;batch_sub_udf_filtered_out=0;batch_sub_lang_read_success=0;batch_sub_lang_write_success=0;batch_sub_lang_delete_success=0;batch_sub_lang_error=0;from_proxy_batch_sub_tsvc_error=0;from_proxy_batch_sub_tsvc_timeout=0;from_proxy_batch_sub_read_success=0;from_proxy_batch_sub_read_error=0;from_proxy_batch_sub_read_timeout=0;from_proxy_batch_sub_read_not_found=0;from_proxy_batch_sub_read_filtered_out=0;from_proxy_batch_sub_write_success=0;from_proxy_batch_sub_write_error=0;from_proxy_batch_sub_write_timeout=0;from_proxy_batch_sub_write_filtered_out=0;from_proxy_batch_sub_delete_success=0;from_proxy_batch_sub_delete_error=0;from_proxy_batch_sub_delete_timeout=0;from_proxy_batch_sub_delete_not_found=0;from_proxy_batch_sub_delete_filtered_out=0;from_proxy_batch_sub_udf_complete=0;from_proxy_batch_sub_udf_error=0;from_proxy_batch_sub_udf_timeout=0;from_proxy_batch_sub_udf_filtered_out=0;from_proxy_batch_sub_lang_read_success=0;from_proxy_batch_sub_lang_write_success=0;from_proxy_batch_sub_lang_delete_success=0;from_proxy_batch_sub_lang_error=0;udf_sub_tsvc_error=0;udf_sub_tsvc_timeout=0;udf_sub_udf_complete=0;udf_sub_udf_error=0;udf_sub_udf_timeout=0;udf_sub_udf_filtered_out=0;udf_sub_lang_read_success=0;udf_sub_lang_write_success=0;udf_sub_lang_delete_success=0;udf_sub_lang_error=0;ops_sub_tsvc_error=0;ops_sub_tsvc_timeout=0;ops_sub_write_success=0;ops_sub_write_error=0;ops_sub_write_timeout=0;ops_sub_write_filtered_out=0;dup_res_ask=0;dup_res_respond_read=0;dup_res_respond_no_read=0;retransmit_all_read_dup_res=0;retransmit_all_write_dup_res=0;retransmit_all_write_repl_write=0;retransmit_all_delete_dup_res=0;retransmit_all_delete_repl_write=0;retransmit_all_udf_dup_res=0;retransmit_all_udf_repl_write=0;retransmit_all_batch_sub_dup_res=0;retransmit_udf_sub_dup_res=0;retransmit_udf_sub_repl_write=0;retransmit_ops_sub_dup_res=0;retransmit_ops_sub_repl_write=0;pi_query_short_basic_complete=0;pi_query_short_basic_error=0;pi_query_short_basic_timeout=0;pi_query_long_basic_complete=0;pi_query_long_basic_error=0;pi_query_long_basic_abort=0;pi_query_aggr_complete=0;pi_query_aggr_error=0;pi_query_aggr_abort=0;pi_query_udf_bg_complete=0;pi_query_udf_bg_error=0;pi_query_udf_bg_abort=0;pi_query_ops_bg_complete=0;pi_query_ops_bg_error=0;pi_query_ops_bg_abort=0;si_query_short_basic_complete=0;si_query_short_basic_error=0;si_query_short_basic_timeout=0;si_query_long_basic_complete=0;si_query_long_basic_error=0;si_query_long_basic_abort=0;si_query_aggr_complete=0;si_query_aggr_error=0;si_query_aggr_abort=0;si_query_udf_bg_complete=0;si_query_udf_bg_error=0;si_query_udf_bg_abort=0;si_query_ops_bg_complete=0;si_query_ops_bg_error=0;si_query_ops_bg_abort=0;geo_region_query_reqs=0;geo_region_query_cells=0;geo_region_query_points=0;geo_region_query_falsepos=0;re_repl_tsvc_error=0;re_repl_tsvc_timeout=0;re_repl_success=0;re_repl_error=0;re_repl_timeout=0;fail_xdr_forbidden=0;fail_key_busy=0;fail_generation=0;fail_record_too_big=0;fail_client_lost_conflict=0;fail_xdr_lost_conflict=0;deleted_last_bin=0;allow-ttl-without-nsup=false;background-query-max-rps=10000;conflict-resolution-policy=generation;conflict-resolve-writes=false;default-ttl=0;disable-cold-start-eviction=false;disable-write-dup-res=false;disallow-expunge=false;disallow-null-setname=false;enable-benchmarks-batch-sub=false;enable-benchmarks-ops-sub=false;enable-benchmarks-read=false;enable-benchmarks-udf=false;enable-benchmarks-udf-sub=false;enable-benchmarks-write=false;enable-hist-proxy=false;evict-hist-buckets=10000;evict-tenths-pct=5;force-long-queries=false;high-water-disk-pct=0;high-water-memory-pct=30;ignore-migrate-fill-delay=false;index-stage-size=1073741824;inline-short-queries=false;max-record-size=0;memory-size=4294967296;migrate-order=5;migrate-retransmit-ms=5000;migrate-sleep=1;nsup-hist-period=3600;nsup-period=0;nsup-threads=1;partition-tree-sprigs=256;prefer-uniform-balance=true;rack-id=0;read-consistency-level-override=off;reject-non-xdr-writes=false;reject-xdr-writes=false;replication-factor=2;sindex-stage-size=1073741824;single-query-threads=4;stop-writes-pct=90;stop-writes-sys-memory-pct=90;strong-consistency=false;strong-consistency-allow-expunge=false;tomb-raider-eligible-age=86400;tomb-raider-period=86400;transaction-pending-limit=20;truncate-threads=4;write-commit-level-override=off;xdr-bin-tombstone-ttl=86400;xdr-tomb-raider-period=120;xdr-tomb-raider-threads=1;geo2dsphere-within.strict=true;geo2dsphere-within.min-level=1;geo2dsphere-within.max-level=20;geo2dsphere-within.max-cells=12;geo2dsphere-within.level-mod=1;geo2dsphere-within.earth-radius-meters=6371000;index-type=flash;index-type.mount[0]=/root/data/bar_flash_disk_mount;index-type.mounts-high-water-pct=30;index-type.mounts-size-limit=4294967296;sindex-type=flash;sindex-type.mount[0]=/root/data/sindex_bar_on_flash;sindex-type.mounts-high-water-pct=0;sindex-type.mounts-size-limit=4294967296;storage-engine=memory +namespace-stats:pmkohl_on_device:ns_cluster_size=1;effective_replication_factor=1;objects=330164;tombstones=0;xdr_tombstones=0;xdr_bin_cemeteries=0;master_objects=330164;master_tombstones=0;prole_objects=0;prole_tombstones=0;non_replica_objects=0;non_replica_tombstones=0;unreplicated_records=0;dead_partitions=0;unavailable_partitions=0;clock_skew_stop_writes=false;stop_writes=false;hwm_breached=false;current_time=427402124;non_expirable_objects=0;expired_objects=0;evicted_objects=0;evict_ttl=0;evict_void_time=0;smd_evict_void_time=0;nsup_cycle_duration=0;nsup_cycle_deleted_pct=0.00;truncate_lut=0;truncating=false;sindex_gc_cleaned=0;memory_used_bytes=0;memory_used_data_bytes=0;memory_used_index_bytes=0;memory_used_set_index_bytes=0;memory_used_sindex_bytes=0;memory_free_pct=100;xmem_id=7;available_bin_names=65532;index_pmem_used_bytes=21130496;index_pmem_used_pct=0;sindex_pmem_used_bytes=0;sindex_pmem_used_pct=0;pmem_total_bytes=4294967296;pmem_used_bytes=40693088;pmem_free_pct=99;pmem_available_pct=98;storage-engine.file[0].used_bytes=40693088;storage-engine.file[0].free_wblocks=506;storage-engine.file[0].write_q=0;storage-engine.file[0].writes=4;storage-engine.file[0].defrag_q=0;storage-engine.file[0].defrag_reads=0;storage-engine.file[0].defrag_writes=0;storage-engine.file[0].shadow_write_q=0;storage-engine.file[0].age=-1;record_proto_uncompressed_pct=0.000;record_proto_compression_ratio=1.000;query_proto_uncompressed_pct=0.000;query_proto_compression_ratio=1.000;pending_quiesce=false;effective_is_quiesced=false;nodes_quiesced=0;effective_prefer_uniform_balance=true;migrate_tx_partitions_imbalance=0;migrate_tx_instances=0;migrate_rx_instances=0;migrate_tx_partitions_active=0;migrate_rx_partitions_active=0;migrate_tx_partitions_initial=0;migrate_tx_partitions_remaining=0;migrate_tx_partitions_lead_remaining=0;migrate_rx_partitions_initial=0;migrate_rx_partitions_remaining=0;migrate_records_skipped=0;migrate_records_transmitted=0;migrate_record_retransmits=0;migrate_record_receives=0;migrate_signals_active=0;migrate_signals_remaining=0;appeals_tx_active=0;appeals_rx_active=0;appeals_tx_remaining=0;appeals_records_exonerated=0;client_tsvc_error=0;client_tsvc_timeout=0;client_proxy_complete=0;client_proxy_error=0;client_proxy_timeout=0;client_read_success=0;client_read_error=0;client_read_timeout=0;client_read_not_found=0;client_read_filtered_out=0;client_write_success=330164;client_write_error=0;client_write_timeout=0;client_write_filtered_out=0;xdr_client_write_success=0;xdr_client_write_error=0;xdr_client_write_timeout=0;client_delete_success=0;client_delete_error=0;client_delete_timeout=0;client_delete_not_found=0;client_delete_filtered_out=0;xdr_client_delete_success=0;xdr_client_delete_error=0;xdr_client_delete_timeout=0;xdr_client_delete_not_found=0;client_udf_complete=0;client_udf_error=0;client_udf_timeout=0;client_udf_filtered_out=0;client_lang_read_success=0;client_lang_write_success=0;client_lang_delete_success=0;client_lang_error=0;from_proxy_tsvc_error=0;from_proxy_tsvc_timeout=0;from_proxy_read_success=0;from_proxy_read_error=0;from_proxy_read_timeout=0;from_proxy_read_not_found=0;from_proxy_read_filtered_out=0;from_proxy_write_success=0;from_proxy_write_error=0;from_proxy_write_timeout=0;from_proxy_write_filtered_out=0;xdr_from_proxy_write_success=0;xdr_from_proxy_write_error=0;xdr_from_proxy_write_timeout=0;from_proxy_delete_success=0;from_proxy_delete_error=0;from_proxy_delete_timeout=0;from_proxy_delete_not_found=0;from_proxy_delete_filtered_out=0;xdr_from_proxy_delete_success=0;xdr_from_proxy_delete_error=0;xdr_from_proxy_delete_timeout=0;xdr_from_proxy_delete_not_found=0;from_proxy_udf_complete=0;from_proxy_udf_error=0;from_proxy_udf_timeout=0;from_proxy_udf_filtered_out=0;from_proxy_lang_read_success=0;from_proxy_lang_write_success=0;from_proxy_lang_delete_success=0;from_proxy_lang_error=0;batch_sub_tsvc_error=0;batch_sub_tsvc_timeout=0;batch_sub_proxy_complete=0;batch_sub_proxy_error=0;batch_sub_proxy_timeout=0;batch_sub_read_success=0;batch_sub_read_error=0;batch_sub_read_timeout=0;batch_sub_read_not_found=0;batch_sub_read_filtered_out=0;batch_sub_write_success=0;batch_sub_write_error=0;batch_sub_write_timeout=0;batch_sub_write_filtered_out=0;batch_sub_delete_success=0;batch_sub_delete_error=0;batch_sub_delete_timeout=0;batch_sub_delete_not_found=0;batch_sub_delete_filtered_out=0;batch_sub_udf_complete=0;batch_sub_udf_error=0;batch_sub_udf_timeout=0;batch_sub_udf_filtered_out=0;batch_sub_lang_read_success=0;batch_sub_lang_write_success=0;batch_sub_lang_delete_success=0;batch_sub_lang_error=0;from_proxy_batch_sub_tsvc_error=0;from_proxy_batch_sub_tsvc_timeout=0;from_proxy_batch_sub_read_success=0;from_proxy_batch_sub_read_error=0;from_proxy_batch_sub_read_timeout=0;from_proxy_batch_sub_read_not_found=0;from_proxy_batch_sub_read_filtered_out=0;from_proxy_batch_sub_write_success=0;from_proxy_batch_sub_write_error=0;from_proxy_batch_sub_write_timeout=0;from_proxy_batch_sub_write_filtered_out=0;from_proxy_batch_sub_delete_success=0;from_proxy_batch_sub_delete_error=0;from_proxy_batch_sub_delete_timeout=0;from_proxy_batch_sub_delete_not_found=0;from_proxy_batch_sub_delete_filtered_out=0;from_proxy_batch_sub_udf_complete=0;from_proxy_batch_sub_udf_error=0;from_proxy_batch_sub_udf_timeout=0;from_proxy_batch_sub_udf_filtered_out=0;from_proxy_batch_sub_lang_read_success=0;from_proxy_batch_sub_lang_write_success=0;from_proxy_batch_sub_lang_delete_success=0;from_proxy_batch_sub_lang_error=0;udf_sub_tsvc_error=0;udf_sub_tsvc_timeout=0;udf_sub_udf_complete=0;udf_sub_udf_error=0;udf_sub_udf_timeout=0;udf_sub_udf_filtered_out=0;udf_sub_lang_read_success=0;udf_sub_lang_write_success=0;udf_sub_lang_delete_success=0;udf_sub_lang_error=0;ops_sub_tsvc_error=0;ops_sub_tsvc_timeout=0;ops_sub_write_success=0;ops_sub_write_error=0;ops_sub_write_timeout=0;ops_sub_write_filtered_out=0;dup_res_ask=0;dup_res_respond_read=0;dup_res_respond_no_read=0;retransmit_all_read_dup_res=0;retransmit_all_write_dup_res=0;retransmit_all_write_repl_write=0;retransmit_all_delete_dup_res=0;retransmit_all_delete_repl_write=0;retransmit_all_udf_dup_res=0;retransmit_all_udf_repl_write=0;retransmit_all_batch_sub_dup_res=0;retransmit_udf_sub_dup_res=0;retransmit_udf_sub_repl_write=0;retransmit_ops_sub_dup_res=0;retransmit_ops_sub_repl_write=0;pi_query_short_basic_complete=0;pi_query_short_basic_error=0;pi_query_short_basic_timeout=0;pi_query_long_basic_complete=0;pi_query_long_basic_error=0;pi_query_long_basic_abort=0;pi_query_aggr_complete=0;pi_query_aggr_error=0;pi_query_aggr_abort=0;pi_query_udf_bg_complete=0;pi_query_udf_bg_error=0;pi_query_udf_bg_abort=0;pi_query_ops_bg_complete=0;pi_query_ops_bg_error=0;pi_query_ops_bg_abort=0;si_query_short_basic_complete=0;si_query_short_basic_error=0;si_query_short_basic_timeout=0;si_query_long_basic_complete=0;si_query_long_basic_error=0;si_query_long_basic_abort=0;si_query_aggr_complete=0;si_query_aggr_error=0;si_query_aggr_abort=0;si_query_udf_bg_complete=0;si_query_udf_bg_error=0;si_query_udf_bg_abort=0;si_query_ops_bg_complete=0;si_query_ops_bg_error=0;si_query_ops_bg_abort=0;geo_region_query_reqs=0;geo_region_query_cells=0;geo_region_query_points=0;geo_region_query_falsepos=0;re_repl_tsvc_error=0;re_repl_tsvc_timeout=0;re_repl_success=0;re_repl_error=0;re_repl_timeout=0;fail_xdr_forbidden=0;fail_key_busy=0;fail_generation=0;fail_record_too_big=0;fail_client_lost_conflict=0;fail_xdr_lost_conflict=0;deleted_last_bin=0;allow-ttl-without-nsup=false;background-query-max-rps=10000;conflict-resolution-policy=generation;conflict-resolve-writes=false;default-ttl=0;disable-cold-start-eviction=false;disable-write-dup-res=false;disallow-expunge=false;disallow-null-setname=false;enable-benchmarks-batch-sub=false;enable-benchmarks-ops-sub=false;enable-benchmarks-read=false;enable-benchmarks-udf=false;enable-benchmarks-udf-sub=false;enable-benchmarks-write=false;enable-hist-proxy=false;evict-hist-buckets=10000;evict-tenths-pct=5;force-long-queries=false;high-water-disk-pct=0;high-water-memory-pct=0;ignore-migrate-fill-delay=false;index-stage-size=1073741824;inline-short-queries=false;max-record-size=0;memory-size=4294967296;migrate-order=5;migrate-retransmit-ms=5000;migrate-sleep=1;nsup-hist-period=3600;nsup-period=0;nsup-threads=1;partition-tree-sprigs=256;prefer-uniform-balance=true;rack-id=0;read-consistency-level-override=off;reject-non-xdr-writes=false;reject-xdr-writes=false;replication-factor=2;sindex-stage-size=1073741824;single-query-threads=4;stop-writes-pct=90;stop-writes-sys-memory-pct=90;strong-consistency=false;strong-consistency-allow-expunge=false;tomb-raider-eligible-age=86400;tomb-raider-period=86400;transaction-pending-limit=20;truncate-threads=4;write-commit-level-override=off;xdr-bin-tombstone-ttl=86400;xdr-tomb-raider-period=120;xdr-tomb-raider-threads=1;geo2dsphere-within.strict=true;geo2dsphere-within.min-level=1;geo2dsphere-within.max-level=20;geo2dsphere-within.max-cells=12;geo2dsphere-within.level-mod=1;geo2dsphere-within.earth-radius-meters=6371000;index-type=pmem;index-type.mount[0]=/root/data/pmem_pmkohl_mount;index-type.mounts-high-water-pct=0;index-type.mounts-size-limit=4294967296;sindex-type=pmem;sindex-type.mount[0]=/root/data/sindex_pmem_pmkohl;sindex-type.mounts-high-water-pct=0;sindex-type.mounts-size-limit=4294967296;storage-engine=pmem;storage-engine.file[0]=/root/data/pmkohl.dat;storage-engine.commit-to-device=false;storage-engine.compression=none;storage-engine.compression-acceleration=0;storage-engine.compression-level=0;storage-engine.defrag-lwm-pct=50;storage-engine.defrag-queue-min=0;storage-engine.defrag-sleep=1000;storage-engine.defrag-startup-minimum=0;storage-engine.direct-files=false;storage-engine.disable-odsync=false;storage-engine.enable-benchmarks-storage=false;storage-engine.encryption-key-file=null;storage-engine.encryption-old-key-file=null;storage-engine.filesize=4294967296;storage-engine.flush-max-ms=1000;storage-engine.max-used-pct=70;storage-engine.max-write-cache=67108864;storage-engine.min-avail-pct=5;storage-engine.serialize-tomb-raider=false;storage-engine.tomb-raider-sleep=1000 +set-stats:[ns=test:set=from_branch_2:objects=0:tombstones=0:memory_data_bytes=0:device_data_bytes=0:truncate_lut=0:sindexes=1:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0 ns=test:set=from_branch_3:objects=3136813:tombstones=0:memory_data_bytes=215797088:device_data_bytes=0:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0 ns=ns_test_on_flash:set=from_branch_1:objects=3136834:tombstones=0:memory_data_bytes=215798539:device_data_bytes=0:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0 ns=ns_test_on_flash:set=ns_test_2:objects=3136826:tombstones=0:memory_data_bytes=215797989:device_data_bytes=0:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0 ns=test_on_shmem:set=shmem_set_1:objects=330161:tombstones=0:memory_data_bytes=22675124:device_data_bytes=0:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0 ns=bar_on_flash:set=bar_flash_set_2:objects=330169:tombstones=0:memory_data_bytes=22675668:device_data_bytes=0:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0 ns=pmkohl_on_device:set=kohl_set_1:objects=330164:tombstones=0:memory_data_bytes=0:device_data_bytes=40693088:truncate_lut=0:sindexes=0:index_populating=false:truncating=false:disable-eviction=false:enable-index=false:stop-writes-count=0:stop-writes-size=0 ] +latency-stats:map[bar_on_flash:map[write:map[bucketLabels:[+Inf 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536] bucketValues:[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] timeUnit:ms]] ns_test_on_flash:map[write:map[bucketLabels:[+Inf 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536] bucketValues:[5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5 5513.5] timeUnit:ms]] pmkohl_on_device:map[write:map[bucketLabels:[+Inf 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536] bucketValues:[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] timeUnit:ms]] test:map[write:map[bucketLabels:[+Inf 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536] bucketValues:[2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9 2755.9] timeUnit:ms]] test_on_shmem:map[write:map[bucketLabels:[+Inf 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536] bucketValues:[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] timeUnit:ms]]] +sindex-stats:test:test_sindex1:entries=0;used_bytes=16777216;entries_per_bval=0;entries_per_rec=0;load_pct=100;load_time=0;stat_gc_recs=0 + +node-configs:advertise-ipv6=false;auto-pin=none;batch-index-threads=5;batch-max-buffers-per-queue=255;batch-max-unused-buffers=256;cluster-name=63flash;debug-allocations=none;disable-udf-execution=false;downgrading=false;enable-benchmarks-fabric=false;enable-health-check=false;enable-hist-info=false;enforce-best-practices=false;feature-key-file[0]=/etc/aerospike/features.conf;indent-allocations=false;info-max-ms=10000;info-threads=16;keep-caps-ssd-health=false;log-local-time=false;log-millis=false;microsecond-histograms=false;migrate-fill-delay=0;migrate-max-num-incoming=4;migrate-threads=1;min-cluster-size=1;node-id=BB9060011AC4202;node-id-interface=null;os-group-perms=false;pidfile=/var/run/aerospike/asd.pid;proto-fd-idle-ms=0;proto-fd-max=15000;query-max-done=100;query-threads-limit=128;run-as-daemon=true;salt-allocations=false;secrets-address-port=null;secrets-tls-context=null;service-threads=5;sindex-builder-threads=4;sindex-gc-period=10;stay-quiesced=false;ticker-interval=10;transaction-max-ms=1000;transaction-retry-ms=1002;vault-ca=null;vault-namespace=null;vault-path=null;vault-token-file=null;vault-url=null;work-directory=/opt/aerospike +node-stats:failed_best_practices=true;cluster_size=1;cluster_key=65E72D696EED;cluster_generation=1;cluster_principal=BB9060011AC4202;cluster_min_compatibility_id=11;cluster_max_compatibility_id=11;cluster_integrity=true;cluster_is_member=true;cluster_duplicate_nodes=null;cluster_clock_skew_stop_writes_sec=0;cluster_clock_skew_ms=0;cluster_clock_skew_outliers=null;uptime=14955;system_total_cpu_pct=21;system_user_cpu_pct=12;system_kernel_cpu_pct=9;system_free_mem_kbytes=8482620;system_free_mem_pct=69;system_thp_mem_kbytes=32768;process_cpu_pct=4;threads_joinable=10;threads_detached=106;threads_pool_total=27;threads_pool_active=27;heap_allocated_kbytes=2234572;heap_active_kbytes=2236196;heap_mapped_kbytes=2330112;heap_efficiency_pct=100;heap_site_count=0;objects=14411347;tombstones=0;info_queue=0;rw_in_progress=0;proxy_in_progress=0;tree_gc_queue=0;client_connections=1;client_connections_opened=4574;client_connections_closed=4573;heartbeat_connections=0;heartbeat_connections_opened=0;heartbeat_connections_closed=0;fabric_connections=0;fabric_connections_opened=0;fabric_connections_closed=0;heartbeat_received_self=0;heartbeat_received_foreign=0;reaped_fds=0;info_complete=8193;info_timeout=0;demarshal_error=0;early_tsvc_client_error=30;early_tsvc_from_proxy_error=0;early_tsvc_batch_sub_error=0;early_tsvc_from_proxy_batch_sub_error=0;early_tsvc_udf_sub_error=0;early_tsvc_ops_sub_error=0;long_queries_active=0;batch_index_initiate=0;batch_index_queue=0:0,0:0,0:0,0:0,0:0;batch_index_complete=0;batch_index_error=0;batch_index_timeout=0;batch_index_delay=0;batch_index_unused_buffers=0;batch_index_huge_buffers=0;batch_index_created_buffers=0;batch_index_destroyed_buffers=0;batch_index_proto_uncompressed_pct=0.000;batch_index_proto_compression_ratio=1.000;paxos_principal=BB9060011AC4202;time_since_rebalance=14949;migrate_allowed=true;migrate_partitions_remaining=0;fabric_bulk_send_rate=0;fabric_bulk_recv_rate=0;fabric_ctrl_send_rate=0;fabric_ctrl_recv_rate=0;fabric_meta_send_rate=0;fabric_meta_recv_rate=0;fabric_rw_send_rate=0;fabric_rw_recv_rate=0 + +xdr-get-config:context=xdr;dc=backup_dc_asdev20:auth-mode=none;auth-password-file=null;auth-user=null;connector=false;max-recoveries-interleaved=0;node-address-port=asdev20:3000;period-ms=100;tls-name=null;use-alternate-access-address=false;namespaces= +xdr-get-config:context=xdr;dc=backup_dc_asdev20_second:auth-mode=none;auth-password-file=null;auth-user=null;connector=false;max-recoveries-interleaved=0;node-address-port=asdev20:3000;period-ms=100;tls-name=null;use-alternate-access-address=false;namespaces= + +xdr-get-config:context=xdr;dc=backup_dc_asdev20_second;namespace=bar_device:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20_second;namespace=bar_on_flash:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20_second;namespace=materials:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20_second;namespace=ns_test_on_flash:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20_second;namespace=pmkohl_on_device:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20_second;namespace=test_on_shmem:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20_second;namespace=test:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto + +xdr-get-config:context=xdr;dc=backup_dc_asdev20;namespace=bar_device:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20;namespace=bar_on_flash:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20;namespace=materials:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20;namespace=ns_test_on_flash:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20;namespace=pmkohl_on_device:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20;namespace=test_on_shmem:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto +xdr-get-config:context=xdr;dc=backup_dc_asdev20;namespace=test:enabled=false;bin-policy=all;compression-level=1;compression-threshold=128;delay-ms=0;enable-compression=false;forward=false;hot-key-ms=100;ignored-bins=;ignore-expunges=false;ignored-sets=;max-throughput=100000;remote-namespace=null;sc-replication-wait-ms=100;shipped-bins=;ship-bin-luts=false;ship-nsup-deletes=false;ship-only-specified-sets=false;shipped-sets=;transaction-queue-limit=16384;write-policy=auto + +xdr-get-stats:context=xdr;dc=backup_dc_asdev20:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;nodes=0;throughput=0;latency_ms=0;lap_us=18 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20_second:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;nodes=0;throughput=0;latency_ms=0;lap_us=17 + +xdr-get-stats:context=xdr;dc=backup_dc_asdev20_second;namespace=bar_device:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20_second;namespace=bar_on_flash:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20_second;namespace=materials:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20_second;namespace=ns_test_on_flash:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20_second;namespace=pmkohl_on_device:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20_second;namespace=test_on_shmem:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20_second;namespace=test:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20;namespace=bar_device:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20;namespace=bar_on_flash:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20;namespace=materials:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20;namespace=ns_test_on_flash:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20;namespace=pmkohl_on_device:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20;namespace=test_on_shmem:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 +xdr-get-stats:context=xdr;dc=backup_dc_asdev20;namespace=test:lag=0;in_queue=0;in_progress=0;success=0;abandoned=0;not_found=0;filtered_out=0;retry_no_node=0;retry_conn_reset=0;retry_dest=0;recoveries=0;recoveries_pending=0;hot_keys=0;bytes_shipped=0;uncompressed_pct=0.000;compression_ratio=1.000;throughput=0 + diff --git a/watcher_latency.go b/watcher_latency.go index dbc9c100..10125525 100644 --- a/watcher_latency.go +++ b/watcher_latency.go @@ -12,7 +12,8 @@ type LatencyWatcher struct { func (lw *LatencyWatcher) describe(ch chan<- *prometheus.Desc) {} func (lw *LatencyWatcher) passOneKeys() []string { - return []string{"build"} + // return []string{"build"} + return nil } func (lw *LatencyWatcher) passTwoKeys(rawMetrics map[string]string) (latencyCommands []string) { diff --git a/watcher_latency_test.go b/watcher_latency_test.go index 6ba68124..559d2b46 100644 --- a/watcher_latency_test.go +++ b/watcher_latency_test.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "os" "strings" "testing" "time" @@ -19,14 +18,17 @@ func TestLatencies_PassOneKeys(t *testing.T) { fmt.Println("TestLatencies_PassOneKeys: outputs: ", passOneKeys) - assert.Equal(t, passOneKeys, []string{"build"}) + // assert.Equal(t, passOneKeys, []string{"build"}) + // modified test-case as watcher-latency no more returns build, this "build" is moved to observer.go + assert.Nil(t, passOneKeys) } func TestLatencies_PassTwoKeys(t *testing.T) { + fmt.Println("initializing config ... TestLatencies_PassTwoKeys") - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) + mas := new(MockAerospikeServer) + mas.initialize() - fmt.Println("initializing config ... TestLatencies_PassTwoKeys") // Initialize and validate config config = new(Config) initConfig(DEFAULT_APE_TOML, config) @@ -34,19 +36,18 @@ func TestLatencies_PassTwoKeys(t *testing.T) { watcher := new(LatencyWatcher) - rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() // simulate, as if we are sending requestInfo to AS and get the Latencies, these are coming from mock-data-generator outputs := watcher.passTwoKeys(rawMetrics) fmt.Println("TestLatencies_PassTwoKeys: outputs: ", outputs) + // output is not nil assert.NotNil(t, outputs, "build details returns are: ", outputs) assert.Equal(t, outputs, []string{"latencies:"}) } func TestLatencies_RefreshDefault(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestLatencies_RefreshDefault") // Initialize and validate config config = new(Config) @@ -56,13 +57,14 @@ func TestLatencies_RefreshDefault(t *testing.T) { // run the test-case logic latencies_runTestCase(t) - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } func TestLatencies_RefreshWithLabelsConfig(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestLatencies_RefreshWithLabelsConfig") + + mas := new(MockAerospikeServer) + mas.initialize() + // Initialize and validate config config = new(Config) initConfig(LABELS_APE_TOML, config) @@ -71,15 +73,13 @@ func TestLatencies_RefreshWithLabelsConfig(t *testing.T) { watcher := new(LatencyWatcher) gaugeStatHandler = new(GaugeStats) - initGaugeStats(METRICS_CONFIG_FILE, gaugeStatHandler) - rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() lObserver := &Observer{} ch := make(chan prometheus.Metric, 1000) latenciesInfoKeys := watcher.passTwoKeys(rawMetrics) - watcher.passTwoKeys(rawMetrics) err := watcher.refresh(lObserver, latenciesInfoKeys, rawMetrics, ch) if err != nil { @@ -123,20 +123,22 @@ func TestLatencies_RefreshWithLabelsConfig(t *testing.T) { * complete logic to call watcher, generate-mock data and asset is part of this function */ func latencies_runTestCase(t *testing.T) { - watcher := new(LatencyWatcher) - gaugeStatHandler = new(GaugeStats) + // mock server + mas := new(MockAerospikeServer) + mas.initialize() + + latdg := new(MockLatencyPromMetricGenerator) + gaugeStatHandler = new(GaugeStats) initGaugeStats(METRICS_CONFIG_FILE, gaugeStatHandler) - rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() + watcher := new(LatencyWatcher) lObserver := &Observer{} - ch := make(chan prometheus.Metric, 1000) - latenciesInfoKeys := watcher.passTwoKeys(rawMetrics) + ch := make(chan prometheus.Metric, 10000) - fmt.Println(" processing latencies with InfoKeys: ", latenciesInfoKeys) - - watcher.passTwoKeys(rawMetrics) + latenciesInfoKeys := watcher.passTwoKeys(rawMetrics) err := watcher.refresh(lObserver, latenciesInfoKeys, rawMetrics, ch) if err != nil { @@ -201,7 +203,7 @@ func latencies_runTestCase(t *testing.T) { // we have only 1 service in our mock-data, however loop thru service array for _, keyValue := range arrServices { - lExpectedMetricNamedValues, lExpectedMetricLabels := createLatencysWatcherExpectedOutputs(keyValue) + lExpectedMetricNamedValues, lExpectedMetricLabels := latdg.createLatencysWatcherExpectedOutputs(mas, keyValue) for key := range lOutputValues { expectedValues := lExpectedMetricNamedValues[key] diff --git a/watcher_namespaces.go b/watcher_namespaces.go index 9e12c52c..9f25f59e 100644 --- a/watcher_namespaces.go +++ b/watcher_namespaces.go @@ -3,12 +3,41 @@ package main import ( "regexp" "strings" + "time" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) +var regexToExtractArrayStats = map[string]string{ + STORAGE_ENGINE: "storage\\-engine\\.(?Pfile|device)\\[(?P\\d+)\\]\\.(?P.+)", + INDEX_TYPE: "index\\-type\\.(?Pmount)\\[(?P\\d+)\\]\\.(?P.+)", + SINDEX_TYPE: "sindex\\-type\\.(?Pmount)\\[(?P\\d+)\\]\\.(?P.+)", +} + +// index-pressure related variables +var ( + // dont fetch 1st iteration, this is made true after reading the metrics once from server and if index-type=false is enabled + isFlashStatSentByServer bool = false + + // time interval to fetch index-pressure + idxPressureFetchInterval = 10.0 + + // Time when Index Pressure last-fetched + idxPressurePreviousFetchTime = time.Now() +) + +const ( + KEY_NS_METADATA = "namespaces" + KEY_NS_INDEX_PRESSURE = "index-pressure" + + KEY_NS_NAMESPACE = "namespace" + + // values to compare while checking if refresh index-pressure or not + TYPE_FLASH = "flash" +) + type NamespaceWatcher struct { namespaceStats map[string]AerospikeStat } @@ -16,92 +45,249 @@ type NamespaceWatcher struct { func (nw *NamespaceWatcher) describe(ch chan<- *prometheus.Desc) {} func (nw *NamespaceWatcher) passOneKeys() []string { - return []string{"namespaces"} + // we are sending key "namespaces", server returns all the configs and stats in single call, unlike node-stats, xdr + return []string{KEY_NS_METADATA} } func (nw *NamespaceWatcher) passTwoKeys(rawMetrics map[string]string) []string { - s := rawMetrics["namespaces"] + s := rawMetrics[KEY_NS_METADATA] list := strings.Split(s, ";") + log.Tracef("namespaces:%s", s) + var infoKeys []string for _, k := range list { - infoKeys = append(infoKeys, "namespace/"+k) + // infoKey ==> namespace/test, namespace/bar + infoKeys = append(infoKeys, KEY_NS_NAMESPACE+"/"+k) + } + + if nw.canSendIndexPressureInfoKey() { + infoKeys = append(infoKeys, KEY_NS_INDEX_PRESSURE) + idxPressurePreviousFetchTime = time.Now() } return infoKeys } -// All (allowed/blocked) namespace stats. Based on the config.Aerospike.NamespaceMetricsAllowlist, config.Aerospike.NamespaceMetricsBlocklist. -// TODO: move this to NamespaceWatcher -- as thie belongs there -// var namespaceStats = make(map[string]AerospikeStat) - -// Regex for identifying storage-engine stats. - func (nw *NamespaceWatcher) refresh(ott *Observer, infoKeys []string, rawMetrics map[string]string, ch chan<- prometheus.Metric) error { - seDynamicExtractor := regexp.MustCompile(`storage\-engine\.(?Pfile|device)\[(?P\d+)\]\.(?P.+)`) if nw.namespaceStats == nil { nw.namespaceStats = make(map[string]AerospikeStat) } - for _, ns := range infoKeys { - nsName := strings.ReplaceAll(ns, "namespace/", "") - log.Tracef("namespace-stats:%s:%s", nsName, rawMetrics[ns]) + for _, infoKey := range infoKeys { + + // we get 2 info-key Types - examples: index-pressure or namespace/test, namespace/materials + if strings.HasPrefix(infoKey, KEY_NS_NAMESPACE) { + nw.refreshNamespaceStats(infoKey, infoKeys, rawMetrics, ch) + } else if strings.HasPrefix(infoKey, KEY_NS_INDEX_PRESSURE) { + // namespace/ will be multiple times according to the # of namespaces configured in the server + nw.refreshIndexPressure(infoKey, infoKeys, rawMetrics, ch) + } + + } + + return nil +} + +// handle IndexPressure infoKey +func (nw *NamespaceWatcher) refreshIndexPressure(singleInfoKey string, infoKeys []string, rawMetrics map[string]string, ch chan<- prometheus.Metric) { + + indexPresssureStats := rawMetrics[singleInfoKey] + + log.Tracef("namespace-index-pressure-stats:%s:%s", singleInfoKey, indexPresssureStats) + + // Server index-pressure output: test:0:0;bar_device:0:0;materials:0:0 + // each namespace is split with semi-colon + stats := strings.Split(indexPresssureStats, ";") + + // metric-names - first element is un-used, as we send namespace as a label, this also keeps the index-numbers same as the server-stats + indexPressureMetricNames := []string{"index_pressure_namespace", "index_pressure_total_memory", "index_pressure_dirty_memory"} - stats := parseStats(rawMetrics[ns], ";") - for stat, value := range stats { + // loop thru each namespace values, + // Server index-pressure output: "test:0:0", "bar_device:0:0" + // refer: https://docs.aerospike.com/reference/info#index-pressure + for _, nsIdxPressureValues := range stats { - pv, err := tryConvert(value) + // refer: https://docs.aerospike.com/reference/info#index-pressure + // each namespace index-pressure stat values separated using colon(:) + // 0= namespace, 1= total memory and 2= memory in dirty-pages + values := strings.Split(nsIdxPressureValues, ":") + nsName := values[0] + + labels := []string{METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE, METRIC_LABEL_NS} + labelValues := []string{rawMetrics[ikClusterName], rawMetrics[ikService], nsName} + + // Server index-pressure output: test:0:0;bar_device:0:0;materials:0:0 + // ignore first element - namespace + for index := 1; index < len(values); index++ { + + pv, err := tryConvert(values[index]) if err != nil { continue } - // handle any panic from prometheus, this may occur when prom encounters a config/stat with special characters - defer func() { - if r := recover(); r != nil { - log.Tracef("namespace-stats: recovered from panic while handling stat %s in %s", stat, nsName) - } - }() - - // to find regular metric or storage-engine metric, we split stat [using: seDynamicExtractor RegEx] - // after splitting, a storage-engine stat has 4 elements other stats have 3 elements - match := seDynamicExtractor.FindStringSubmatch(stat) - - // process storage engine stat - if len(match) == 4 { - statType := match[1] - statIndex := match[2] - statName := match[3] - - compositeStatName := STORAGE_ENGINE + statType + "_" + statName - asMetric, exists := nw.namespaceStats[compositeStatName] - - if !exists { - asMetric = newAerospikeStat(CTX_NAMESPACE, compositeStatName) - nw.namespaceStats[compositeStatName] = asMetric - } - - if asMetric.isAllowed { - deviceOrFileName := stats["storage-engine."+statType+"["+statIndex+"]"] - - desc, valueType := asMetric.makePromMetric(METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE, METRIC_LABEL_NS, statType+"_index", statType) - ch <- prometheus.MustNewConstMetric(desc, valueType, pv, rawMetrics[ikClusterName], rawMetrics[ikService], nsName, statIndex, deviceOrFileName) - } - } else { // regular stat (i.e. non-storage-engine related) - asMetric, exists := nw.namespaceStats[stat] - - if !exists { - asMetric = newAerospikeStat(CTX_NAMESPACE, stat) - nw.namespaceStats[stat] = asMetric - } - - if asMetric.isAllowed { - desc, valueType := asMetric.makePromMetric(METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE, METRIC_LABEL_NS) - ch <- prometheus.MustNewConstMetric(desc, valueType, pv, rawMetrics[ikClusterName], rawMetrics[ikService], nsName) - } + statName := indexPressureMetricNames[index] + asMetric, exists := nw.namespaceStats[statName] + if !exists { + asMetric = newAerospikeStat(CTX_NAMESPACE, statName) + nw.namespaceStats[statName] = asMetric } + // push to prom-channel + pushToPrometheus(asMetric, pv, labels, labelValues, ch) } } - return nil +} + +// all namespace stats (except index-pressure) +func (nw *NamespaceWatcher) refreshNamespaceStats(singleInfoKey string, infoKeys []string, rawMetrics map[string]string, ch chan<- prometheus.Metric) { + + // extract namespace from info-command, construct: namespace/test, namespace/bar + nsName := strings.ReplaceAll(singleInfoKey, (KEY_NS_NAMESPACE + "/"), "") + + log.Tracef("namespace-stats:%s:%s", nsName, rawMetrics[singleInfoKey]) + + stats := parseStats(rawMetrics[singleInfoKey], ";") + var labels []string + var labelValues []string + constructedStatname := "" + + for stat, value := range stats { + + pv, err := tryConvert(value) + if err != nil { + continue + } + + // check persistance-type + deviceType, isArrayType := nw.checkStatPersistanceType(stat, stats) + + // default: aerospike_namespace_ + constructedStatname = stat + labels = []string{METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE, METRIC_LABEL_NS} + labelValues = []string{rawMetrics[ikClusterName], rawMetrics[ikService], nsName} + + if isArrayType { + constructedStatname, labels, labelValues = nw.handleArrayStats(nsName, stat, pv, stats, deviceType, rawMetrics, ch) + } + + // check and include persistance-type if they are defined/found + indexType := stats[INDEX_TYPE] + sindexType := stats[SINDEX_TYPE] + + // if stat is index-type or sindex-type , append addl label + if strings.HasPrefix(deviceType, INDEX_TYPE) && len(indexType) > 0 { + labels = append(labels, METRIC_LABEL_INDEX) + labelValues = append(labelValues, indexType) + + // check is it flash or pmem or shmem + nw.setFlagFlashStatSentByServer(indexType) + + } else if strings.HasPrefix(deviceType, SINDEX_TYPE) && len(sindexType) > 0 { + labels = append(labels, METRIC_LABEL_SINDEX) + labelValues = append(labelValues, sindexType) + + // check is it flash or pmem or shmem + nw.setFlagFlashStatSentByServer(sindexType) + } + + // handleArrayStats(..) will return empty-string if unable to handle the array-stat + if len(constructedStatname) > 0 { + asMetric, exists := nw.namespaceStats[constructedStatname] + + if !exists { + asMetric = newAerospikeStat(CTX_NAMESPACE, constructedStatname) + nw.namespaceStats[constructedStatname] = asMetric + } + + // push to prom-channel + pushToPrometheus(asMetric, pv, labels, labelValues, ch) + } + } + +} + +// Utility to handle array style stats like storage-engine or index-type etc., +// example: storage-device..file[0].defrag_q +// +// Each part of the stat is split into 4 groups using a regex, +// each value of the 4 groups represents type like stats-type, index-number, sub-stat-name (like file[0].age) +// - example: group[0]= , group[1]= stat-type, group[2]= array-index, group[3]= sub-stat-name +func (nw *NamespaceWatcher) handleArrayStats(nsName string, statToProcess string, pv float64, + allNamespaceStats map[string]string, deviceType string, rawMetrics map[string]string, + ch chan<- prometheus.Metric) (string, []string, []string) { + + regexStr := regexToExtractArrayStats[deviceType] + dynamicExtractor := regexp.MustCompile(regexStr) + + // to find regular metric or storage-engine metric, we split stat [using: seDynamicExtractor RegEx] + // after splitting, a index-type/storage-engine stat has 4 elements other stats have 3 elements + + match := dynamicExtractor.FindStringSubmatch(statToProcess) + if len(match) != 4 { + log.Warnf("namespace-stats: stat %s in unexpected format, length is not 4 as per regex", statToProcess) + return "", nil, nil + } + + statType := match[1] + statIndex := match[2] + statName := match[3] + + compositeStatName := deviceType + "_" + statType + "_" + statName + deviceOrFileName := allNamespaceStats[deviceType+"."+statType+"["+statIndex+"]"] + labels := []string{METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE, METRIC_LABEL_NS, statType + "_index", statType} + labelValues := []string{rawMetrics[ikClusterName], rawMetrics[ikService], nsName, statIndex, deviceOrFileName} + + return compositeStatName, labels, labelValues + +} + +// Utility functions used within namespace-watcher + +// checks if the given stat is a storage-engine or index-type, depending on which type we decide how to process +// +// multiple values are returnd by server example: storage-engine.file[0]. +func (nw *NamespaceWatcher) checkStatPersistanceType(statToProcess string, + allNamespaceStats map[string]string) (string, bool) { + + // if starts-with "index-type", then + // return index-type, + + if strings.HasPrefix(statToProcess, INDEX_TYPE) { + return INDEX_TYPE, strings.Contains(statToProcess, "[") + } else if strings.HasPrefix(statToProcess, SINDEX_TYPE) { + return SINDEX_TYPE, strings.Contains(statToProcess, "[") + } else if strings.HasPrefix(statToProcess, STORAGE_ENGINE) { + return STORAGE_ENGINE, strings.Contains(statToProcess, "[") + } + + return "", false +} + +// utility function to check if watcher-namespace needs to issue infoKeys command during passTwo +// index-pressure is a costly command at server side hence we are limiting to every few minutes ( mentioned in seconds) +func (nw *NamespaceWatcher) canSendIndexPressureInfoKey() bool { + + // difference between current-time and last-fetch, if its > defined-value, then true + timeDiff := time.Since(idxPressurePreviousFetchTime) + + // if index-type=false or sindex-type=flash is returned by server + // and every N seconds - where N is mentioned "indexPressureFetchIntervalInSeconds" + isTimeOk := timeDiff.Minutes() >= idxPressureFetchInterval + + return (isFlashStatSentByServer && isTimeOk) + +} + +// utility will check if the given value is flash and sets the flag +func (nw *NamespaceWatcher) setFlagFlashStatSentByServer(idxType string) { + // index-type/sindex-type, can have values like shmem(default), flash, so if the value is flash, then set the flag to refresh in next run + // this check is required only if bool-fetch-indexpressure is false, because + // we can have different values for each namespace, so once this flag is set, no need to change this further + // + if len(idxType) > 0 && !isFlashStatSentByServer { + isFlashStatSentByServer = !isFlashStatSentByServer && strings.Contains(idxType, TYPE_FLASH) + } + } diff --git a/watcher_namespaces_test.go b/watcher_namespaces_test.go index 3a3f026d..cbd66af5 100644 --- a/watcher_namespaces_test.go +++ b/watcher_namespaces_test.go @@ -2,17 +2,24 @@ package main import ( "fmt" - "os" "strings" "testing" "time" "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/assert" - dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" ) +func TestFetchNamespaces(t *testing.T) { + + mas := new(MockAerospikeServer) + mas.initialize() + namespaces := mas.requestInfoNamespaces() + + assert.NotEmpty(t, namespaces) +} + func TestPassOneKeys(t *testing.T) { nsWatcher := new(NamespaceWatcher) // Check passoneKeys @@ -22,29 +29,32 @@ func TestPassOneKeys(t *testing.T) { } func TestPassTwoKeys(t *testing.T) { - rawMetrics := getRawMetrics() + mas := new(MockAerospikeServer) + mas.initialize() + + // rawMetrics := getRawMetrics() nsWatcher := new(NamespaceWatcher) // simulate, as if we are sending requestInfo to AS and get the namespaces, these are coming from mock-data-generator - pass2Keys := requestInfoNamespaces(rawMetrics) - nsOutputs := nsWatcher.passTwoKeys(pass2Keys) + passOneKeyOutputs := mas.requestInfoNamespaces() + passTwokeyOutputs := nsWatcher.passTwoKeys(passOneKeyOutputs) - // nsExpected := []string{"namespace/bar", "namespace/test"} - nsExpected := createNamespacePassTwoExpectedOutputs(rawMetrics) + // expectedOutputs := []string{"namespace/bar", "namespace/test"} + expectedOutputs := mas.createNamespacePassTwoExpectedOutputs() assert := assert.New(t) - for idx := range nsExpected { + assert.NotEmpty(passTwokeyOutputs) + assert.NotEmpty(expectedOutputs) + + for idx := range expectedOutputs { // assert each element returned by NamespaceWatcher exists in expected outputs - assert.Contains(nsOutputs, nsExpected[idx], " value exists!") + assert.Contains(passTwokeyOutputs, expectedOutputs[idx], " value exists!") } } func TestNamespaceRefreshDefault(t *testing.T) { - // this is to force-reload the config in the NamespaceWatcher, this is a check on this param in NamespaceWatcher implementation - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestNamespaceRefreshDefault") // Initialize and validate config config = new(Config) @@ -53,14 +63,10 @@ func TestNamespaceRefreshDefault(t *testing.T) { config.validateAndUpdate() runTestcase(t) - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } func TestNamespaceRefreshLabels(t *testing.T) { - // this is to force-reload the config in the NamespaceWatcher, this is a check on this param in NamespaceWatcher implementation - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestNamespaceRefreshLabels") // Initialize and validate config config = new(Config) @@ -69,14 +75,9 @@ func TestNamespaceRefreshLabels(t *testing.T) { config.validateAndUpdate() runTestcase(t) - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } func TestNamespaceRefreshAllowlist(t *testing.T) { - - // this is to force-reload the config in the NamespaceWatcher, this is a check on this param in NamespaceWatcher implementation - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestNamespaceRefreshAllowlist") // Initialize and validate config config = new(Config) @@ -89,10 +90,6 @@ func TestNamespaceRefreshAllowlist(t *testing.T) { } func TestNamespaceRefreshBlocklist(t *testing.T) { - - // this is to force-reload the config in the NamespaceWatcher, this is a check on this param in NamespaceWatcher implementation - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestNamespaceRefreshBlocklist") // Initialize and validate config config = new(Config) @@ -109,25 +106,30 @@ func TestNamespaceRefreshBlocklist(t *testing.T) { */ func runTestcase(t *testing.T) { - gaugeStatHandler = new(GaugeStats) + // mock aerospike server + mas := new(MockAerospikeServer) + mas.initialize() + // mock namespace prom metric generator + nsdg := new(MockNamespacePromMetricGenerator) + + // initialize gauges list + gaugeStatHandler = new(GaugeStats) initGaugeStats(METRICS_CONFIG_FILE, gaugeStatHandler) // read raw-metrics from mock data gen, create observer and channel prometeus metric ingestion and processing - rawMetrics := getRawMetrics() + // rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() + + // Actual Watcher-Namespace generator code nsWatcher := new(NamespaceWatcher) lObserver := &Observer{} - ch := make(chan prometheus.Metric, 1000) - pass2Metrics := requestInfoNamespaces(rawMetrics) - - nsWatcher.passTwoKeys(rawMetrics) - - nsInfoKeys := createNamespacePassTwoExpectedOutputs(rawMetrics) + ch := make(chan prometheus.Metric, 10000) - // outputs := nsWatcher.passTwoKeys(pass2Metrics) - // assert.Equal(t, outputs, expectedOutputs) + passOneKeyOutputs := mas.requestInfoNamespaces() + passTwokeyOutputs := nsWatcher.passTwoKeys(passOneKeyOutputs) - err := nsWatcher.refresh(lObserver, nsInfoKeys, rawMetrics, ch) + err := nsWatcher.refresh(lObserver, passTwokeyOutputs, rawMetrics, ch) if err == nil { // map of string ==> map["namespace/metric-name"][""] @@ -139,6 +141,8 @@ func runTestcase(t *testing.T) { // reads data from the Prom channel and creates a map of strings so we can assert in the below loop domore := 1 + // mock_expected_filedump := []string{} + for domore == 1 { select { @@ -161,12 +165,19 @@ func runTestcase(t *testing.T) { // Desc{fqName: "aerospike_namespac_memory_free_pct", help: "memory free pct", constLabels: {}, variableLabels: [cluster_name service ns]} metricNameFromDesc := extractMetricNameFromDesc(description) namespaceFromLabel := extractNamespaceFromLabel(metricLabel) + // namespaceFromLabel := extractLabelNameValueFromFullLabel(metricLabel) + labelString := stringifyLabel(metricLabel) // key will be like namespace/, this we use this check during assertion - keyName := makeKeyname(namespaceFromLabel, metricNameFromDesc, true) + keyName := makeKeyname(namespaceFromLabel, labelString, true) + keyName = makeKeyname(keyName, metricNameFromDesc, true) + lOutputValues[keyName] = metricValue lOutputLabels[keyName] = metricLabel + // to dump expected outputs to a file, so no-need-to regenerate every-time + // mock_expected_filedump = append(mock_expected_filedump, makeMetricReady2dump(namespaceFromLabel, metricNameFromDesc, description, metricValue)) + case <-time.After(1 * time.Second): domore = 0 @@ -174,20 +185,21 @@ func runTestcase(t *testing.T) { } // loop each namespace and compare the label and value - arrNames := strings.Split(pass2Metrics["namespaces"], ";") + arrNames := strings.Split(passOneKeyOutputs["namespaces"], ";") for nsIndex := range arrNames { tnsForNamespace := arrNames[nsIndex] - lExpectedMetricNamedValues, lExpectedMetricLabels := createNamespaceWatcherExpectedOutputs(tnsForNamespace, true) + lExpectedMetricNamedValues, lExpectedMetricLabels := nsdg.createNamespaceWatcherExpectedOutputs(mas, tnsForNamespace, true) for key := range lOutputValues { expectedValues := lExpectedMetricNamedValues[key] expectedLabels := lExpectedMetricLabels[key] outputMetricValues := lOutputValues[key] outpuMetricLabels := lOutputLabels[key] - // assert - only if the value belongs to the namespace we read expected values and processing - if strings.HasPrefix(key, tnsForNamespace) { + // a "/" because namespace can be like test and test_on_shmem, + if strings.HasPrefix(key, tnsForNamespace+"/") { + assert.Contains(t, expectedValues, outputMetricValues) assert.Contains(t, expectedLabels, outpuMetricLabels) } @@ -197,4 +209,10 @@ func runTestcase(t *testing.T) { } else { fmt.Println(" Failed Refreshing, error: ", err) } + } + +// to dump expected outputs to a file, so no-need-to regenerate every-time +// func makeMetricReady2dump(ns string, metric string, desc string, value string) string { +// return ns + "#" + metric + "#" + desc + "#" + value +// } diff --git a/watcher_node_stats.go b/watcher_node_stats.go index 7c8620fc..6ab99b62 100644 --- a/watcher_node_stats.go +++ b/watcher_node_stats.go @@ -6,6 +6,11 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + KEY_SERVICE_CONFIG = "get-config:context=service" + KEY_SERVICE_STATISTICS = "statistics" +) + type StatsWatcher struct { nodeMetrics map[string]AerospikeStat } @@ -17,7 +22,8 @@ func (sw *StatsWatcher) passOneKeys() []string { } func (sw *StatsWatcher) passTwoKeys(rawMetrics map[string]string) []string { - return []string{"statistics"} + // we need to fetch both configs and stat + return []string{KEY_SERVICE_CONFIG, KEY_SERVICE_STATISTICS} } // All (allowed/blocked) node stats. Based on the config.Aerospike.NodeMetricsAllowlist, config.Aerospike.NodeMetricsBlocklist. @@ -25,13 +31,32 @@ func (sw *StatsWatcher) passTwoKeys(rawMetrics map[string]string) []string { func (sw *StatsWatcher) refresh(o *Observer, infoKeys []string, rawMetrics map[string]string, ch chan<- prometheus.Metric) error { - log.Tracef("node-stats:%s", rawMetrics["statistics"]) - if sw.nodeMetrics == nil { sw.nodeMetrics = make(map[string]AerospikeStat) } - stats := parseStats(rawMetrics["statistics"], ";") + nodeConfigs := rawMetrics[KEY_SERVICE_CONFIG] + nodeStats := rawMetrics[KEY_SERVICE_STATISTICS] + log.Tracef("node-configs:%s", nodeConfigs) + log.Tracef("node-stats:%s", nodeStats) + + clusterName := rawMetrics[ikClusterName] + service := rawMetrics[ikService] + + // we are sending configs and stats in same refresh call, as both are being sent to prom, instead of doing prom-push in 2 functions + // handle configs + sw.handleRefresh(o, nodeConfigs, clusterName, service, ch) + + // handle stats + sw.handleRefresh(o, nodeStats, clusterName, service, ch) + + return nil +} + +func (sw *StatsWatcher) handleRefresh(o *Observer, nodeRawMetrics string, clusterName string, service string, + ch chan<- prometheus.Metric) { + + stats := parseStats(nodeRawMetrics, ";") for stat, value := range stats { pv, err := tryConvert(value) @@ -45,18 +70,10 @@ func (sw *StatsWatcher) refresh(o *Observer, infoKeys []string, rawMetrics map[s sw.nodeMetrics[stat] = asMetric } - // handle any panic from prometheus, this may occur when prom encounters a config/stat with special characters - defer func() { - if r := recover(); r != nil { - log.Tracef("node-stats: recovered from panic while handling stat %s in %s", stat, gService) - } - }() + labels := []string{METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE} + labelsValues := []string{clusterName, service} - if asMetric.isAllowed { - desc, valueType := asMetric.makePromMetric(METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE) - ch <- prometheus.MustNewConstMetric(desc, valueType, pv, rawMetrics[ikClusterName], rawMetrics[ikService]) - } - } + pushToPrometheus(asMetric, pv, labels, labelsValues, ch) - return nil + } } diff --git a/watcher_node_stats_test.go b/watcher_node_stats_test.go index 8b77931a..d918d31b 100644 --- a/watcher_node_stats_test.go +++ b/watcher_node_stats_test.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "os" "strings" "testing" "time" @@ -13,6 +12,8 @@ import ( ) func TestNodeStats_PassOneKeys(t *testing.T) { + fmt.Println("initializing config ... TestNodeStats_PassOneKeys") + watcher := new(StatsWatcher) // Check passoneKeys passOneKeys := watcher.passOneKeys() @@ -21,20 +22,18 @@ func TestNodeStats_PassOneKeys(t *testing.T) { } func TestNodeStats_PassTwoKeys(t *testing.T) { + fmt.Println("initializing config ... TestNodeStats_PassTwoKeys") + watcher := new(StatsWatcher) // simulate, as if we are sending requestInfo to AS and get the NodeStats, these are coming from mock-data-generator pass2Keys := make(map[string]string) outputs := watcher.passTwoKeys(pass2Keys) - fmt.Println("TestNodeStats_PassTwoKeys: outputs: ", outputs) - - assert.Equal(t, outputs, []string{"statistics"}) + assert.Equal(t, outputs, []string{"get-config:context=service", "statistics"}) } func TestNodeStats_RefreshDefault(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestNodeStats_RefreshDefault") // Initialize and validate config config = new(Config) @@ -44,13 +43,9 @@ func TestNodeStats_RefreshDefault(t *testing.T) { // run the test-case logic nodeStats_runTestCase(t) - - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } func TestNodeStats_Allowlist(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestNodeStats_Allowlist") // Initialize and validate config config = new(Config) @@ -60,13 +55,9 @@ func TestNodeStats_Allowlist(t *testing.T) { // run the test-case logic nodeStats_runTestCase(t) - - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } func TestNodeStats_Blocklist(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestNodeStats_Blocklist") // Initialize and validate config config = new(Config) @@ -77,24 +68,30 @@ func TestNodeStats_Blocklist(t *testing.T) { // run the test-case logic nodeStats_runTestCase(t) - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } /** * complete logic to call watcher, generate-mock data and asset is part of this function */ func nodeStats_runTestCase(t *testing.T) { - watcher := new(StatsWatcher) - gaugeStatHandler = new(GaugeStats) + // mock aerospike server + mas := new(MockAerospikeServer) + mas.initialize() + + // mock node-stats prom metric generator + nstdg := new(MockNodestatPromMetricGenerator) + gaugeStatHandler = new(GaugeStats) initGaugeStats(METRICS_CONFIG_FILE, gaugeStatHandler) - rawMetrics := getRawMetrics() + // rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() lObserver := &Observer{} - ch := make(chan prometheus.Metric, 1000) + ch := make(chan prometheus.Metric, 10000) nodeStatsInfoKeys := []string{} + watcher := new(StatsWatcher) watcher.passTwoKeys(rawMetrics) err := watcher.refresh(lObserver, nodeStatsInfoKeys, rawMetrics, ch) @@ -152,7 +149,7 @@ func nodeStats_runTestCase(t *testing.T) { for serviceIndex := range arrServices { serviceIp := arrServices[serviceIndex] - lExpectedMetricNamedValues, lExpectedMetricLabels := createNodeStatsWatcherExpectedOutputs(serviceIp) + lExpectedMetricNamedValues, lExpectedMetricLabels := nstdg.createNodeStatsWatcherExpectedOutputs(mas, serviceIp) for key := range lOutputValues { expectedValues := lExpectedMetricNamedValues[key] @@ -162,6 +159,7 @@ func nodeStats_runTestCase(t *testing.T) { // assert - only if the value belongs to the service we read expected values and processing if strings.HasPrefix(key, serviceIp) { + assert.Contains(t, expectedValues, outputMetricValues) assert.Contains(t, expectedLabels, outpuMetricLabels) } diff --git a/watcher_sets.go b/watcher_sets.go index e1e6d688..cd5da6ef 100644 --- a/watcher_sets.go +++ b/watcher_sets.go @@ -22,9 +22,6 @@ func (sw *SetWatcher) passTwoKeys(rawMetrics map[string]string) []string { return []string{"sets"} } -// All (allowed/blocked) Sets stats. Based on the config.Aerospike.SetsMetricsAllowlist, config.Aerospike.SetsMetricsBlocklist. -// var setMetrics = make(map[string]AerospikeStat) - func (sw *SetWatcher) refresh(o *Observer, infoKeys []string, rawMetrics map[string]string, ch chan<- prometheus.Metric) error { setStats := strings.Split(rawMetrics["sets"], ";") log.Tracef("set-stats:%v", setStats) @@ -34,6 +31,8 @@ func (sw *SetWatcher) refresh(o *Observer, infoKeys []string, rawMetrics map[str } for i := range setStats { + clusterName := rawMetrics[ikClusterName] + service := rawMetrics[ikService] stats := parseStats(setStats[i], ":") for stat, value := range stats { @@ -48,17 +47,9 @@ func (sw *SetWatcher) refresh(o *Observer, infoKeys []string, rawMetrics map[str sw.setMetrics[stat] = asMetric } - // handle any panic from prometheus, this may occur when prom encounters a config/stat with special characters - defer func() { - if r := recover(); r != nil { - log.Tracef("set-stats: recovered from panic while handling stat %s in %s", stat, stats["set"]) - } - }() - - if asMetric.isAllowed { - desc, valueType := asMetric.makePromMetric(METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE, METRIC_LABEL_NS, METRIC_LABEL_SET) - ch <- prometheus.MustNewConstMetric(desc, valueType, pv, rawMetrics[ikClusterName], rawMetrics[ikService], stats["ns"], stats["set"]) - } + labels := []string{METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE, METRIC_LABEL_NS, METRIC_LABEL_SET} + labelsValues := []string{clusterName, service, stats["ns"], stats["set"]} + pushToPrometheus(asMetric, pv, labels, labelsValues, ch) } diff --git a/watcher_sets_test.go b/watcher_sets_test.go index 31171541..bee86baa 100644 --- a/watcher_sets_test.go +++ b/watcher_sets_test.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "os" "strings" "testing" "time" @@ -13,6 +12,8 @@ import ( ) func TestSets_PassOneKeys(t *testing.T) { + fmt.Println("initializing config ... TestSets_PassOneKeys") + watcher := new(SetWatcher) // Check passoneKeys passOneKeys := watcher.passOneKeys() @@ -21,20 +22,21 @@ func TestSets_PassOneKeys(t *testing.T) { } func TestSets_PassTwoKeys(t *testing.T) { + fmt.Println("initializing config ... TestSets_PassTwoKeys") + watcher := new(SetWatcher) + // mock aerospike server + mas := new(MockAerospikeServer) + mas.initialize() + rawMetrics := mas.fetchRawMetrics() // simulate, as if we are sending requestInfo to AS and get the NodeStats, these are coming from mock-data-generator - pass2Keys := make(map[string]string) - outputs := watcher.passTwoKeys(pass2Keys) - - fmt.Println("TestNodeStats_PassTwoKeys: outputs: ", outputs) + outputs := watcher.passTwoKeys(rawMetrics) assert.Equal(t, outputs, []string{"sets"}) } func TestSets_RefreshDefault(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestSets_RefreshDefault") // Initialize and validate config config = new(Config) @@ -44,13 +46,9 @@ func TestSets_RefreshDefault(t *testing.T) { // run the test-case logic sets_runTestCase(t) - - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } func TestSets_Allowlist(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestSets_Allowlist") // Initialize and validate config config = new(Config) @@ -60,14 +58,14 @@ func TestSets_Allowlist(t *testing.T) { // run the test-case logic sets_runTestCase(t) - - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } func TestSets_RefreshWithLabelsConfig(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestSets_RefreshWithLabelsConfig") + + mas := new(MockAerospikeServer) + mas.initialize() + // Initialize and validate config config = new(Config) initConfig(LABELS_APE_TOML, config) @@ -78,10 +76,10 @@ func TestSets_RefreshWithLabelsConfig(t *testing.T) { gaugeStatHandler = new(GaugeStats) initGaugeStats(METRICS_CONFIG_FILE, gaugeStatHandler) - rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() lObserver := &Observer{} - ch := make(chan prometheus.Metric, 1000) + ch := make(chan prometheus.Metric, 10000) setsInfoKeys := []string{} watcher.passTwoKeys(rawMetrics) @@ -128,12 +126,20 @@ func TestSets_RefreshWithLabelsConfig(t *testing.T) { * complete logic to call watcher, generate-mock data and asset is part of this function */ func sets_runTestCase(t *testing.T) { + + // mock aerospike server + mas := new(MockAerospikeServer) + mas.initialize() + + msdg := new(MockSetsPromMetricGenerator) + watcher := new(SetWatcher) gaugeStatHandler = new(GaugeStats) initGaugeStats(METRICS_CONFIG_FILE, gaugeStatHandler) - rawMetrics := getRawMetrics() + // rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() lObserver := &Observer{} ch := make(chan prometheus.Metric, 1000) @@ -198,7 +204,7 @@ func sets_runTestCase(t *testing.T) { // we have only 1 service in our mock-data, however loop thru service array for _, namespaceWithSetName := range arrNamespaceSets { - lExpectedMetricNamedValues, lExpectedMetricLabels := createSetsWatcherExpectedOutputs(namespaceWithSetName) + lExpectedMetricNamedValues, lExpectedMetricLabels := msdg.createSetsWatcherExpectedOutputs(mas, namespaceWithSetName) for key := range lOutputValues { expectedValues := lExpectedMetricNamedValues[key] diff --git a/watcher_sindex.go b/watcher_sindex.go index 1d863019..bf30562c 100644 --- a/watcher_sindex.go +++ b/watcher_sindex.go @@ -45,9 +45,6 @@ func (siw *SindexWatcher) getSindexCommands(sindexesMeta []string) (sindexComman return sindexCommands } -// All (allowed/blocked) Sindex stats. Based on the config.Aerospike.SindexMetricsAllowlist, config.Aerospike.SindexMetricsBlocklist. -// var sindexMetrics = make(map[string]AerospikeStat) - func (siw *SindexWatcher) refresh(o *Observer, infoKeys []string, rawMetrics map[string]string, ch chan<- prometheus.Metric) error { if config.Aerospike.DisableSindexMetrics { // disabled @@ -65,6 +62,9 @@ func (siw *SindexWatcher) refresh(o *Observer, infoKeys []string, rawMetrics map sindexName := sindexInfoKeySplit[1] log.Tracef("sindex-stats:%s:%s:%s", nsName, sindexName, rawMetrics[sindex]) + clusterName := rawMetrics[ikClusterName] + service := rawMetrics[ikService] + stats := parseStats(rawMetrics[sindex], ";") for stat, value := range stats { pv, err := tryConvert(value) @@ -77,17 +77,10 @@ func (siw *SindexWatcher) refresh(o *Observer, infoKeys []string, rawMetrics map asMetric = newAerospikeStat(CTX_SINDEX, stat) siw.sindexMetrics[stat] = asMetric } - // handle any panic from prometheus, this may occur when prom encounters a config/stat with special characters - defer func() { - if r := recover(); r != nil { - log.Tracef("sindex-stats: recovered from panic while handling stat %s in %s", stat, sindexName) - } - }() - - if asMetric.isAllowed { - desc, valueType := asMetric.makePromMetric(METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE, METRIC_LABEL_NS, METRIC_LABEL_SINDEX) - ch <- prometheus.MustNewConstMetric(desc, valueType, pv, rawMetrics[ikClusterName], rawMetrics[ikService], nsName, sindexName) - } + + labels := []string{METRIC_LABEL_CLUSTER_NAME, METRIC_LABEL_SERVICE, METRIC_LABEL_NS, METRIC_LABEL_SINDEX} + labelsValues := []string{clusterName, service, nsName, sindexName} + pushToPrometheus(asMetric, pv, labels, labelsValues, ch) } diff --git a/watcher_sindex_test.go b/watcher_sindex_test.go index 3653a9a5..541eaf43 100644 --- a/watcher_sindex_test.go +++ b/watcher_sindex_test.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "os" "strings" "testing" "time" @@ -28,23 +27,25 @@ func TestSindex_PassOneKeys(t *testing.T) { } func TestSindex_PassTwoKeys(t *testing.T) { + fmt.Println("initializing config ... TestSindex_PassTwoKeys") + watcher := new(SindexWatcher) // simulate, as if we are sending requestInfo to AS and get the NodeStats, these are coming from mock-data-generator - rawMetrics := getRawMetrics() - msdg := new(MockSindexDataGen) - expectedOutputs := msdg.createSindexPassTwoExpectedOutputs(rawMetrics) + // rawMetrics := getRawMetrics() + // mock aerospike server + mas := new(MockAerospikeServer) + mas.initialize() - outputs := watcher.passTwoKeys(rawMetrics) + // rawMetrics := mas.fetchRawMetrics() - fmt.Println("TestSindex_PassTwoKeys: outputs: ", outputs) + expectedOutputs := mas.createSindexPassTwoExpectedOutputs(mas) + outputs := watcher.passTwoKeys(mas.passone_outputs_map) assert.Equal(t, outputs, expectedOutputs) } func TestSindex_RefreshDefault(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestSindex_RefreshDefault") // Initialize and validate config config = new(Config) @@ -54,13 +55,9 @@ func TestSindex_RefreshDefault(t *testing.T) { // run the test-case logic sindex_runTestCase(t) - - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } func TestSindex_Allowlist(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) - fmt.Println("initializing config ... TestSets_Allowlist") // Initialize and validate config config = new(Config) @@ -70,12 +67,11 @@ func TestSindex_Allowlist(t *testing.T) { // run the test-case logic sindex_runTestCase(t) - - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_FALSE) } func TestSindex_RefreshWithLabelsConfig(t *testing.T) { - os.Setenv(TESTCASE_MODE, TESTCASE_MODE_TRUE) + mas := new(MockAerospikeServer) + mas.initialize() fmt.Println("initializing config ... TestSindex_RefreshWithLabelsConfig") // Initialize and validate config @@ -88,7 +84,8 @@ func TestSindex_RefreshWithLabelsConfig(t *testing.T) { gaugeStatHandler = new(GaugeStats) initGaugeStats(METRICS_CONFIG_FILE, gaugeStatHandler) - rawMetrics := getRawMetrics() + // rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() lObserver := &Observer{} ch := make(chan prometheus.Metric, 1000) @@ -139,12 +136,17 @@ func TestSindex_RefreshWithLabelsConfig(t *testing.T) { * complete logic to call watcher, generate-mock data and asset is part of this function */ func sindex_runTestCase(t *testing.T) { + + mas := new(MockAerospikeServer) + mas.initialize() + watcher := new(SindexWatcher) gaugeStatHandler = new(GaugeStats) initGaugeStats(METRICS_CONFIG_FILE, gaugeStatHandler) - rawMetrics := getRawMetrics() + // rawMetrics := getRawMetrics() + rawMetrics := mas.fetchRawMetrics() lObserver := &Observer{} ch := make(chan prometheus.Metric, 1000) @@ -158,7 +160,7 @@ func sindex_runTestCase(t *testing.T) { } else { domore := 1 - msdg := new(MockSindexDataGen) + msdg := new(MockSindexPromMetricGenerator) // map of string ==> map["namespace/metric-name"]["