From 4e0f2747e1687bd44a7b1733b14a82dfbc59afb3 Mon Sep 17 00:00:00 2001 From: Matthew Johnson Date: Thu, 13 Jun 2024 22:16:45 -0400 Subject: [PATCH] fix monotonic counter implementations (#90) This change fixes a couple of assumptions that were made in #87 and #88. These changes were made to address a usage issue for monotonic counters in the spectator-go thin client library. For that library, a common use case is to sample uint64 monotonic data sources, so the data type for that meter type in the library was set to uint64. This lead to a need to cascade that change into spectatord. However, changing the existing data type for `C` from `double` to `uint64_t` caused the monotonic counter usage in atlas-system-agent to start partially failing on parsing numbers. When we made these changes, we overlooked the fact that the `double` data type is mostly used when monotonic data sources need to be converted into base units for recording values. So, a monotonic data source recording microsecond values would have to be divided by 1000, and this results in a fractional number, which is not parsed by `strtoull`. Example error message: ``` While parsing sys.pressure.some,xatlas.process=atlas-system-agent,id=io:117.094624: Got 117 parsing value, ignoring chars starting at .094624 ``` The correct way to support all of these use cases, and to preserve backwards compatibility, is to introduce a new meter type `MonotonicCounterUint` which supports the `uint64_t` data type and handles rollovers, while reverting the changes to `MonotonicCounters` (`C`) and `MonotonicSampled` (`X`). In reverting the changes, we kept the original implemenation of disallowing negative value updates, because there is not a use case where that should happen. We chose not to create a `MonotonicSampledUint` at this point in time, because this is an experimental meter type and there is not currently any demand for it. This strategy mirrors the spectator-java implementation: * https://www.javadoc.io/static/com.netflix.spectator/spectator-api/1.7.13/com/netflix/spectator/api/patterns/PolledMeter.Builder.html#monitorMonotonicCounter-T-java.util.function.ToLongFunction- * https://www.javadoc.io/static/com.netflix.spectator/spectator-api/1.7.13/com/netflix/spectator/api/patterns/PolledMeter.Builder.html#monitorMonotonicCounterDouble-T-java.util.function.ToDoubleFunction- This change updates the admin server `/metrics` endpoint, so that the new meter type will be reflected in the output. A `stats` block is also added to the payload to help make it easier to get a sense of the number of meters that are in use. --- README.md | 40 ++--- admin/admin_server.cc | 30 ++++ admin/admin_server_test.cc | 15 +- server/spectatord.cc | 106 ++++++------- server/spectatord_test.cc | 184 ++++++++++++++--------- spectator/CMakeLists.txt | 2 + spectator/monotonic_counter.cc | 25 +-- spectator/monotonic_counter.h | 7 +- spectator/monotonic_counter_test.cc | 34 ++--- spectator/monotonic_counter_uint.cc | 43 ++++++ spectator/monotonic_counter_uint.h | 21 +++ spectator/monotonic_counter_uint_test.cc | 113 ++++++++++++++ spectator/monotonic_sampled.cc | 31 ++-- spectator/monotonic_sampled.h | 11 +- spectator/monotonic_sampled_test.cc | 36 ++--- spectator/registry.cc | 10 ++ spectator/registry.h | 89 +++++++---- 17 files changed, 534 insertions(+), 263 deletions(-) create mode 100644 spectator/monotonic_counter_uint.cc create mode 100644 spectator/monotonic_counter_uint.h create mode 100644 spectator/monotonic_counter_uint_test.cc diff --git a/README.md b/README.md index a007f7d..7de37af 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ metric lifetimes, and route metrics to the correct backend. spectatord --help spectatord: A daemon that listens for metrics and reports them to Atlas. + --admin_port (Port number for the admin server.); default: 1234; --age_gauge_limit (The maximum number of age gauges that may be reported by this process.); default: 1000; --common_tags (Common tags: nf.app=app,nf.cluster=cluster. Override the @@ -28,11 +29,15 @@ spectatord: A daemon that listens for metrics and reports them to Atlas. --enable_statsd (Enable statsd support.); default: false; --meter_ttl (Meter TTL: expire meters after this period of inactivity.); default: 15m; + --no_common_tags (No common tags will be provided for metrics. Since no + common tags are available, no internal status metrics will be recorded. + Only use this feature for special cases where it is absolutely necessary + to override common tags such as nf.app, and only use it with a secondary + spectatord process.); default: false; --port (Port number for the UDP socket.); default: 1234; --socket_path (Path to the UNIX domain socket.); default: "/run/spectatord/spectatord.unix"; --statsd_port (Port number for the statsd socket.); default: 8125; - --admin_port (Port number for the admin server.); default: 1234; --uri (Optional override URI for the aggregator.); default: ""; --verbose (Use verbose logging.); default: false; --verbose_http (Output debug info for HTTP requests.); default: false; @@ -89,22 +94,23 @@ echo -e "t:server.requestLatency:0.042\nd:server.responseSizes:1024" | nc -u -w0 ### Metric Types -| Symbol | Metric Type | Description | -|--------|-----------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `c` | Counter | The value is the number of increments that have occurred since the last time it was recorded. | -| `d` | Distribution Summary | The value tracks the distribution of events. It is similar to a Timer, but more general, because the size does not have to be a period of time.

For example, it can be used to measure the payload sizes of requests hitting a server or the number of records returned from a query. | -| `g` | Gauge | The value is a number that was sampled at a point in time. The default time-to-live (TTL) for gauges is 900 seconds (15 minutes) - they will continue reporting the last value set for this duration of time.

Optionally, the TTL may be specified in seconds, with a minimum TTL of 5 seconds. For example, `g,120:gauge:42.0` spcifies a gauge with a 120 second (2 minute) TTL. | -| `m` | Max Gauge | The value is a number that was sampled at a point in time, but it is reported as a maximum gauge value to the backend. | -| `t` | Timer | The value is the number of seconds that have elapsed for an event. | -| `A` | Age Gauge | The value is the time in seconds since the epoch at which an event has successfully occurred, or `0` to use the current time in epoch seconds. After an Age Gauge has been set, it will continue reporting the number of seconds since the last time recorded, for as long as the spectatord process runs. The purpose of this metric type is to enable users to more easily implement the Time Since Last Success alerting pattern.

To set a specific time as the last success: `A:time.sinceLastSuccess:1611081000`.

To set `now()` as the last success: `A:time.sinceLastSuccess:0`.

By default, a maximum of `1000` Age Gauges are allowed per `spectatord` process, because there is no mechanism for cleaning them up. This value may be tuned with the `--age_gauge_limit` flag on the spectatord binary. | -| `C` | Monotonic Counter | The value is a monotonically increasing number. A minimum of two samples must be received in order for `spectatord` to calculate a delta value and report it to the backend. The value should be a `uint64` data type, and it will handle rollovers.

A variety of networking metrics may be reported monotonically and this metric type provides a convenient means of recording these values, at the expense of a slower time-to-first metric. | -| `D` | Percentile Distribution Summary | The value tracks the distribution of events, with percentile estimates. It is similar to a Percentile Timer, but more general, because the size does not have to be a period of time.

For example, it can be used to measure the payload sizes of requests hitting a server or the number of records returned from a query.

In order to maintain the data distribution, they have a higher storage cost, with a worst-case of up to 300X that of a standard Distribution Summary. Be diligent about any additional dimensions added to Percentile Distribution Summaries and ensure that they have a small bounded cardinality. | -| `T` | Percentile Timer | The value is the number of seconds that have elapsed for an event, with percentile estimates.

This metric type will track the data distribution by maintaining a set of Counters. The distribution can then be used on the server side to estimate percentiles, while still allowing for arbitrary slicing and dicing based on dimensions.

In order to maintain the data distribution, they have a higher storage cost, with a worst-case of up to 300X that of a standard Timer. Be diligent about any additional dimensions added to Percentile Timers and ensure that they have a small bounded cardinality. | -| `X` | Monotonic Counter with Millisecond Timestamps | The value is a monotonically increasing number, sampled at a specified number of milliseconds since the epoch. A minimum of two samples must be received in order for `spectatord` to calculate a delta value and report it to the backend. The value should be a `uint64` data type, and it will handle rollovers.

This is an experimental metric type that can be used to track monotonic sources that were sampled in the recent past, with the value normalized over the reported time period.

The timestamp in milliseconds since the epoch when the value was sampled must be included as a metric option: `X,1543160297100:monotonic.Source:42` | - -The data type for all numbers except `C` and `X` is `double`. The `C` and `X` values are recorded as `uint64_t`, and -the calculated deltas are passed to the backend as `double`. Passing negative values for `uint64_t` data types will -cause the parsed string value to rollover. +| Symbol | Metric Type | Description | +|--------|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `A` | Age Gauge | The value is the time in seconds since the epoch at which an event has successfully occurred, or `0` to use the current time in epoch seconds. After an Age Gauge has been set, it will continue reporting the number of seconds since the last time recorded, for as long as the spectatord process runs. The purpose of this metric type is to enable users to more easily implement the Time Since Last Success alerting pattern.

To set a specific time as the last success: `A:time.sinceLastSuccess:1611081000`.

To set `now()` as the last success: `A:time.sinceLastSuccess:0`.

By default, a maximum of `1000` Age Gauges are allowed per `spectatord` process, because there is no mechanism for cleaning them up. This value may be tuned with the `--age_gauge_limit` flag on the spectatord binary. | +| `c` | Counter | The value is the number of increments that have occurred since the last time it was recorded. The value will be reported to the backend as a rate-per-second. | +| `C` | Monotonic Counter (double) | The value is a monotonically increasing number. A minimum of two samples must be received in order for `spectatord` to calculate a delta value and report it to the backend as a rate-per-second. The value is a `double` data type, and negative deltas are ignored. This data type provides flexibility for transforming values into base units with division.

Commonly used with networking metrics. | +| `d` | Distribution Summary | The value tracks the distribution of events. It is similar to a Timer, but more general, because the size does not have to be a period of time.

For example, it can be used to measure the payload sizes of requests hitting a server or the number of records returned from a query. | +| `D` | Percentile Distribution Summary | The value tracks the distribution of events, with percentile estimates. It is similar to a Percentile Timer, but more general, because the size does not have to be a period of time.

For example, it can be used to measure the payload sizes of requests hitting a server or the number of records returned from a query.

In order to maintain the data distribution, they have a higher storage cost, with a worst-case of up to 300X that of a standard Distribution Summary. Be diligent about any additional dimensions added to Percentile Distribution Summaries and ensure that they have a small bounded cardinality. | +| `g` | Gauge | The value is a number that was sampled at a point in time. The default time-to-live (TTL) for gauges is 900 seconds (15 minutes) - they will continue reporting the last value set for this duration of time.

Optionally, the TTL may be specified in seconds, with a minimum TTL of 5 seconds. For example, `g,120:gauge:42.0` spcifies a gauge with a 120 second (2 minute) TTL. | +| `m` | Max Gauge | The value is a number that was sampled at a point in time, but it is reported as a maximum gauge value to the backend. | +| `t` | Timer | The value is the number of seconds that have elapsed for an event. | +| `T` | Percentile Timer | The value is the number of seconds that have elapsed for an event, with percentile estimates.

This metric type will track the data distribution by maintaining a set of Counters. The distribution can then be used on the server side to estimate percentiles, while still allowing for arbitrary slicing and dicing based on dimensions.

In order to maintain the data distribution, they have a higher storage cost, with a worst-case of up to 300X that of a standard Timer. Be diligent about any additional dimensions added to Percentile Timers and ensure that they have a small bounded cardinality. | +| `U` | Monotonic Counter (uint64) | The value is a monotonically increasing number. A minimum of two samples must be received in order for `spectatord` to calculate a delta value and report it to the backend as a rate-per-second. The value is a `uint64` data type, and it will handle rollovers.

Commonly used with networking metrics. | +| `X` | Monotonic Counter (double) with Millisecond Timestamps | The value is a monotonically increasing number, sampled at a specified number of milliseconds since the epoch. A minimum of two samples must be received in order for `spectatord` to calculate a delta value and report it to the backend. The value should be a `uint64` data type, and it will handle rollovers.

This is an experimental metric type that can be used to track monotonic sources that were sampled in the recent past, with the value normalized over the reported time period.

The timestamp in milliseconds since the epoch when the value was sampled must be included as a metric option: `X,1543160297100:monotonic.Source:42` | + +The data type for all numbers except `U` is `double`. The `U` values are recorded as `uint64_t`, and the calculated +deltas are passed to the backend as `double`. Passing negative values for `uint64_t` data types will cause the parsed +string value to rollover. ### Metric Name and Tags diff --git a/admin/admin_server.cc b/admin/admin_server.cc index d4ede26..a5bcad6 100644 --- a/admin/admin_server.cc +++ b/admin/admin_server.cc @@ -207,6 +207,14 @@ void GET_metrics(HTTPServerRequest& req, HTTPServerResponse& res, spectator::Reg } obj->set("mono_counters", mono_counters); + Poco::JSON::Array::Ptr mono_counters_uint = new Poco::JSON::Array(true); + for (auto it : registry.MonotonicCountersUint()) { + auto meter = fmt_meter_object((spectator::Meter*)it); + meter->set("value", fmt::format("{}", it->Delta())); + mono_counters_uint->add(meter); + } + obj->set("mono_counters_uint", mono_counters_uint); + Poco::JSON::Array::Ptr timers = new Poco::JSON::Array(true); for (auto it : registry.Timers()) { auto meter = fmt_meter_object((spectator::Meter*)it); @@ -215,6 +223,28 @@ void GET_metrics(HTTPServerRequest& req, HTTPServerResponse& res, spectator::Reg } obj->set("timers", timers); + Object::Ptr stats = new Object(true); + auto age_gauges_size = registry.AgeGauges().size(); + auto counters_size = registry.Counters().size(); + auto dist_summaries_size = registry.DistSummaries().size(); + auto gauges_size = registry.Gauges().size(); + auto max_gauges_size = registry.MaxGauges().size(); + auto mono_counters_size = registry.MonotonicCounters().size(); + auto mono_counters_uint_size = registry.MonotonicCountersUint().size(); + auto timers_size = registry.Timers().size(); + auto total = age_gauges_size + counters_size + dist_summaries_size + gauges_size + + max_gauges_size + mono_counters_size + mono_counters_uint_size + timers_size; + stats->set("age_gauges.size", age_gauges_size); + stats->set("counters.size", counters_size); + stats->set("dist_summaries.size", dist_summaries_size); + stats->set("gauges.size", gauges_size); + stats->set("max_gauges.size", max_gauges_size); + stats->set("mono_counters.size", mono_counters_size); + stats->set("mono_counters_uint.size", mono_counters_uint_size); + stats->set("timers.size", timers_size); + stats->set("total.size", total); + obj->set("stats", stats); + res.setStatus(HTTPResponse::HTTP_OK); res.setContentType("application/json"); obj->stringify(res.send()); diff --git a/admin/admin_server_test.cc b/admin/admin_server_test.cc index 3eea7a8..9bcbb86 100644 --- a/admin/admin_server_test.cc +++ b/admin/admin_server_test.cc @@ -203,16 +203,17 @@ TEST_F(AdminServerTest, GET_metrics) { Var result {parser.parse(rr)}; Object::Ptr object = result.extract(); - int count = 0; - std::vector expected_keys {"age_gauges", "counters", "dist_summaries", - "gauges", "max_gauges", "mono_counters", "timers"}; + std::vector expected_keys {"age_gauges", "counters", "dist_summaries", "gauges", + "max_gauges", "mono_counters", "mono_counters_uint", + "stats", "timers"}; + std::vector found_keys; + for (auto &it : *object) { - if (contains(expected_keys, it.first)) { - count += 1; - } + found_keys.emplace_back(it.first); } + std::sort(found_keys.begin(), found_keys.end()); - EXPECT_EQ(count, expected_keys.size()); + EXPECT_EQ(found_keys, expected_keys); } TEST_F(AdminServerTest, GET_undefined_route) { diff --git a/server/spectatord.cc b/server/spectatord.cc index c06055e..3a45107 100644 --- a/server/spectatord.cc +++ b/server/spectatord.cc @@ -110,19 +110,27 @@ static void update_statsd_metric(spectator::Registry* registry, } } -// parse a single line of the form: -// :||@|#:, -// See: -// https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics -// examples: -// custom_metric:60|g|#shell - Set the custom_metric gauge to 60 and tag it -// with 'shell' page.views:1|c - Increment the page.views COUNT metric. -// fuel.level:0.5|g - Record the fuel tank is half-empty. -// song.length:240|h|@0.5 - Sample the song.length histogram half of the time. -// users.uniques:1234|s - Track unique visitors to the site. -// users.online:1|c|#country:china - Increment the active users COUNT metric -// and tag by country of origin. users.online:1|c|@0.5|#country:china - Track -// active China users and use a sample rate. +/* StatsD Protocol Parser Specification + * + * Parse a single line, of the following form: + * + * :||@|#:, + * + * Reference Link: + * + * https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics + * + * Examples: + * + * custom_metric:60|g|#shell - Set the custom_metric gauge to 60 and tag it + * with 'shell' page.views:1|c - Increment the page.views COUNT metric. + * fuel.level:0.5|g - Record the fuel tank is half-empty. + * song.length:240|h|@0.5 - Sample the song.length histogram half of the time. + * users.uniques:1234|s - Track unique visitors to the site. + * users.online:1|c|#country:china - Increment the active users COUNT metric and tag by + * country of origin. + * users.online:1|c|@0.5|#country:china - Track active China users and use a sample rate. + */ auto Server::parse_statsd_line(const char* buffer) -> std::optional { assert(buffer != nullptr); @@ -151,7 +159,6 @@ auto Server::parse_statsd_line(const char* buffer) char char_type = *p; switch (char_type) { case 'c': - // counter type = StatsdMetricType::Counter; break; case 'g': @@ -254,7 +261,7 @@ auto get_measurement(char type, std::string_view measurement_str, std::string* e auto value_str = measurement_str.begin() + pos; char* last_char = nullptr; valueT value{}; - if (type == 'C' || type == 'X') { + if (type == 'U') { value.u = std::strtoull(value_str, &last_char, 10); } else { value.d = std::strtod(value_str, &last_char); @@ -265,7 +272,7 @@ auto get_measurement(char type, std::string_view measurement_str, std::string* e return {}; } if (*last_char != '\0' && std::isspace(*last_char) == 0) { - if (type == 'C' || type == 'X') { + if (type == 'U') { *err_msg = fmt::format("Got {} parsing value, ignoring chars starting at {}", value.u, last_char); } else { @@ -307,8 +314,7 @@ Server::Server(int port_number, std::optional statsd_port_number, static void prepare_socket_path(const std::string& socket_path) { ::unlink(socket_path.c_str()); - // TODO: Migrate to std::filesystem after we verify it works on all - // our platforms + // TODO: Migrate to std::filesystem after we verify it works on all our platforms auto last = socket_path.rfind('/'); if (last != std::string::npos) { auto dir = socket_path.substr(0, last); @@ -317,8 +323,7 @@ static void prepare_socket_path(const std::string& socket_path) { ::mkdir(dir.c_str(), 0777); } - // We don't want to restrict the permissions on the socket path - // so any user can send metrics + // We don't want to restrict the permissions on the socket path so any user can send metrics umask(0); } @@ -530,57 +535,54 @@ auto Server::parse_line(const char* buffer) -> std::optional { Logger()->info("While parsing {}: {}", p, err_msg); } switch (type) { - case 't': - // timer, elapsed time is reported in seconds - { - auto nanos = static_cast(measurement->value.d * 1e9); - registry_->GetTimer(measurement->id) - ->Record(std::chrono::nanoseconds(nanos)); + case 'A': + if (measurement->value.d == 0) { + registry_->GetAgeGauge(measurement->id)->UpdateLastSuccess(); + } else { + registry_->GetAgeGauge(measurement->id)->UpdateLastSuccess( + static_cast(measurement->value.d * 1e9)); } break; case 'c': - // counter registry_->GetCounter(measurement->id)->Add(measurement->value.d); break; case 'C': - // monotonic counters - registry_->GetMonotonicCounter(measurement->id)->Set(measurement->value.u); + registry_->GetMonotonicCounter(measurement->id)->Set(measurement->value.d); + break; + case 'd': + registry_->GetDistributionSummary(measurement->id)->Record(measurement->value.d); + break; + case 'D': + perc_ds_.get_or_create(registry_, measurement->id)->Record( + static_cast(measurement->value.d)); break; case 'g': - // gauge if (extra > 0) { - registry_->GetGauge(measurement->id, absl::Seconds(extra)) - ->Set(measurement->value.d); + registry_->GetGauge(measurement->id, absl::Seconds(extra))->Set(measurement->value.d); } else { - // this preserves the previous Ttl, otherwise we would override it - // with the default value if we use the previous constructor + // this preserves the previous ttl, otherwise we would override it + // with the default value, if we use the previous constructor registry_->GetGauge(measurement->id)->Set(measurement->value.d); } break; case 'm': registry_->GetMaxGauge(measurement->id)->Update(measurement->value.d); break; - case 'A': - if (measurement->value.d == 0) { - registry_->GetAgeGauge(measurement->id)->UpdateLastSuccess(); - } else { - registry_->GetAgeGauge(measurement->id) - ->UpdateLastSuccess(static_cast(measurement->value.d * 1e9)); + case 't': // elapsed time is reported in seconds + { + auto nanos = static_cast(measurement->value.d * 1e9); + registry_->GetTimer(measurement->id)->Record(std::chrono::nanoseconds(nanos)); } break; - case 'd': - // dist summary - registry_->GetDistributionSummary(measurement->id) - ->Record(measurement->value.d); + case 'T': + { + auto nanos = static_cast(measurement->value.d * 1e9); + perc_timers_.get_or_create(registry_, measurement->id)->Record( + std::chrono::nanoseconds(nanos)); + } break; - case 'T': { - auto nanos = static_cast(measurement->value.d * 1e9); - perc_timers_.get_or_create(registry_, measurement->id) - ->Record(std::chrono::nanoseconds(nanos)); - } break; - case 'D': - perc_ds_.get_or_create(registry_, measurement->id) - ->Record(static_cast(measurement->value.d)); + case 'U': + registry_->GetMonotonicCounterUint(measurement->id)->Set(measurement->value.u); break; case 'X': if (extra > 0) { diff --git a/server/spectatord_test.cc b/server/spectatord_test.cc index 239b7a9..2db21bd 100644 --- a/server/spectatord_test.cc +++ b/server/spectatord_test.cc @@ -87,14 +87,6 @@ void test_statsd(char* buffer, const std::map& expected) { } } -TEST(Spectatord, Statsd_Ctr) { - char_ptr ctr{strdup("page.views:1|c\n")}; - std::map expected{ - {"spectatord.parsedCount|statistic=count", 1}, - {"page.views|statistic=count", 1.0}}; - test_statsd(ctr.get(), expected); -} - TEST(Spectatord, Statsd_Multiline) { // end without a newline char_ptr ctr{strdup("page.views:1|c\npage.views:3|c\npage.views:5|c")}; @@ -112,6 +104,22 @@ TEST(Spectatord, Statsd_Multiline) { test_statsd(ctr.get(), expected); } +TEST(Spectatord, Statsd_Counter) { + char_ptr ctr{strdup("page.views:1|c\n")}; + std::map expected{ + {"spectatord.parsedCount|statistic=count", 1}, + {"page.views|statistic=count", 1.0}}; + test_statsd(ctr.get(), expected); +} + +TEST(Spectatord, Statsd_CounterTags) { + char_ptr c{strdup("users.online:10|c|#country:china")}; + std::map expected{ + {"spectatord.parsedCount|statistic=count", 1}, + {"users.online|country=china|statistic=count", 10}}; + test_statsd(c.get(), expected); +} + TEST(Spectatord, Statsd_GaugeNoTags) { char_ptr g{strdup("fuel.level:0.5|g")}; std::map expected{ @@ -120,7 +128,7 @@ TEST(Spectatord, Statsd_GaugeNoTags) { test_statsd(g.get(), expected); } -TEST(Spectatord, Statsd_GaugeTagsDlft) { +TEST(Spectatord, Statsd_GaugeDefaultTags) { char_ptr g{strdup("custom.metric:60|g|#shell")}; std::map expected{ {"spectatord.parsedCount|statistic=count", 1}, @@ -128,7 +136,7 @@ TEST(Spectatord, Statsd_GaugeTagsDlft) { test_statsd(g.get(), expected); } -TEST(Spectatord, Statsd_HistogramAsDs) { +TEST(Spectatord, Statsd_HistogramAsDistSummary) { char_ptr h{strdup("song.length:240|h|#region:east")}; std::map expected{ {"spectatord.parsedCount|statistic=count", 1}, @@ -150,7 +158,7 @@ TEST(Spectatord, Statsd_SampledHistogram) { test_statsd(h.get(), expected); } -TEST(Spectatord, DISABLED_SetCardinality) { +TEST(Spectatord, DISABLED_Statsd_SetCardinality) { char_ptr s{strdup("users.uniques:1234|s")}; std::map expected{ {"spectatord.parsedCount|statistic=count", 1}, @@ -158,14 +166,6 @@ TEST(Spectatord, DISABLED_SetCardinality) { test_statsd(s.get(), expected); } -TEST(Spectatord, Statsd_CounterTags) { - char_ptr c{strdup("users.online:10|c|#country:china")}; - std::map expected{ - {"spectatord.parsedCount|statistic=count", 1}, - {"users.online|country=china|statistic=count", 10}}; - test_statsd(c.get(), expected); -} - TEST(Spectatord, Statsd_Timer) { char_ptr timer{strdup("req.latency:350|ms|#country:us")}; std::map expected{ @@ -188,6 +188,16 @@ TEST(Spectatord, Statsd_SampledTimer) { test_statsd(timer.get(), expected); } +TEST(Spectatord, ParseNoTags) { + auto logger = Logger(); + char_ptr line{strdup("n:1")}; + std::string err_msg; + auto measurement = *get_measurement('c', line.get(), &err_msg); + logger->info("Got {} = {}", measurement.id, measurement.value.d); + EXPECT_DOUBLE_EQ(measurement.value.d, 1.0); + EXPECT_EQ(measurement.id, spectator::Id("n", spectator::Tags{})); +} + TEST(Spectatord, ParseOneTag) { auto logger = Logger(); char_ptr line{strdup("my.name,foo=bar:42.0")}; @@ -213,16 +223,6 @@ TEST(Spectatord, ParseMultipleTags) { EXPECT_TRUE(err_msg.empty()); } -TEST(Spectatord, ParseNoTags) { - auto logger = Logger(); - char_ptr line{strdup("n:1")}; - std::string err_msg; - auto measurement = *get_measurement('c', line.get(), &err_msg); - logger->info("Got {} = {}", measurement.id, measurement.value.d); - EXPECT_DOUBLE_EQ(measurement.value.d, 1.0); - EXPECT_EQ(measurement.id, spectator::Id("n", spectator::Tags{})); -} - TEST(Spectatord, ParseMissingName) { auto logger = Logger(); char_ptr line{strdup(":1")}; @@ -254,21 +254,6 @@ TEST(Spectatord, ParseIgnoringStuffAtTheEnd) { EXPECT_FALSE(err_msg.empty()); } -TEST(Spectatord, ParseCounter) { - auto logger = Logger(); - spectator::Registry registry{GetConfiguration(), logger}; - test_server server{®istry}; - - char_ptr line1{strdup("c:counter.name:10")}; - char_ptr line2{strdup("c:counter.name:10")}; - server.parse_msg(line1.get()); - server.parse_msg(line2.get()); - - auto map = server.measurements(); - EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 2); - EXPECT_DOUBLE_EQ(map["counter.name|statistic=count"], 20); -} - TEST(Spectatord, ParseMultiline) { auto logger = Logger(); spectator::Registry registry{GetConfiguration(), logger}; @@ -283,23 +268,6 @@ TEST(Spectatord, ParseMultiline) { EXPECT_DOUBLE_EQ(map["counter.name|statistic=count"], 42); } -TEST(Spectatord, ParseTimer) { - auto logger = Logger(); - spectator::Registry registry{GetConfiguration(), logger}; - test_server server{®istry}; - - char_ptr line1{strdup("t:timer.name:.001")}; - char_ptr line2{strdup("t:timer.name:.001")}; - server.parse_msg(line1.get()); - server.parse_msg(line2.get()); - - auto map = server.measurements(); - EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 2); - EXPECT_DOUBLE_EQ(map["timer.name|statistic=count"], 2); - EXPECT_DOUBLE_EQ(map["timer.name|statistic=totalTime"], 0.002); - EXPECT_DOUBLE_EQ(map["timer.name|statistic=max"], 0.001); -} - TEST(Spectatord, ParseAgeGauge) { auto logger = Logger(); auto cfg = GetConfiguration(); @@ -353,6 +321,39 @@ TEST(Spectatord, TooManyAgeGauges) { EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 1); } +TEST(Spectatord, ParseCounter) { + auto logger = Logger(); + spectator::Registry registry{GetConfiguration(), logger}; + test_server server{®istry}; + + char_ptr line1{strdup("c:counter.name:10")}; + char_ptr line2{strdup("c:counter.name:10")}; + server.parse_msg(line1.get()); + server.parse_msg(line2.get()); + + auto map = server.measurements(); + EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 2); + EXPECT_DOUBLE_EQ(map["counter.name|statistic=count"], 20); +} + +TEST(Spectatord, ParseDistSummary) { + auto logger = Logger(); + spectator::Registry registry{GetConfiguration(), logger}; + test_server server{®istry}; + + char_ptr line{strdup("d:dist.summary:1")}; + server.parse_msg(line.get()); + + char_ptr line2{strdup("d:dist.summary:2")}; + server.parse_msg(line2.get()); + + auto map = server.measurements(); + EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 2); + EXPECT_DOUBLE_EQ(map["dist.summary|statistic=count"], 2); + EXPECT_DOUBLE_EQ(map["dist.summary|statistic=totalAmount"], 3); + EXPECT_DOUBLE_EQ(map["dist.summary|statistic=max"], 2); +} + TEST(Spectatord, ParseGauge) { auto logger = Logger(); spectator::Registry registry{GetConfiguration(), logger}; @@ -394,21 +395,68 @@ TEST(Spectatord, ParseGaugeTtl) { EXPECT_EQ(registry.GetGauge("gauge.name")->GetTtl(), absl::Seconds(15)); } -TEST(Spectatord, ParseDistSummary) { +TEST(Spectatord, ParseMonoCounter) { auto logger = Logger(); spectator::Registry registry{GetConfiguration(), logger}; test_server server{®istry}; - char_ptr line{strdup("d:dist.summary:1")}; - server.parse_msg(line.get()); + char_ptr line1{strdup("C:mono_counter.name:10.5")}; + server.parse_msg(line1.get()); - char_ptr line2{strdup("d:dist.summary:2")}; + auto map = server.measurements(); + EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 1); + EXPECT_DOUBLE_EQ(map["mono_counter.name|statistic=count"], 0); + + char_ptr line2{strdup("C:mono_counter.name:20.75")}; + server.parse_msg(line2.get()); + + map = server.measurements(); + EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 1); + EXPECT_DOUBLE_EQ(map["mono_counter.name|statistic=count"], 10.25); +} + +TEST(Spectatord, ParseMonoCounterUint) { + auto logger = Logger(); + spectator::Registry registry{GetConfiguration(), logger}; + test_server server{®istry}; + + char_ptr line1{strdup("U:mono_counter_uint.name:10")}; + server.parse_msg(line1.get()); + + auto map = server.measurements(); + EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 1); + EXPECT_DOUBLE_EQ(map["mono_counter_uint.name|statistic=count"], 0); + + char_ptr line2{strdup("U:mono_counter_uint.name:20")}; + server.parse_msg(line2.get()); + + map = server.measurements(); + EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 1); + EXPECT_DOUBLE_EQ(map["mono_counter_uint.name|statistic=count"], 10); + + // fractional numbers are truncated + char_ptr line3{strdup("U:mono_counter_uint.name:21.5")}; + server.parse_msg(line3.get()); + + map = server.measurements(); + EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 1); + EXPECT_DOUBLE_EQ(map["mono_counter_uint.name|statistic=count"], 1); +} + +TEST(Spectatord, ParseTimer) { + auto logger = Logger(); + spectator::Registry registry{GetConfiguration(), logger}; + test_server server{®istry}; + + char_ptr line1{strdup("t:timer.name:.001")}; + char_ptr line2{strdup("t:timer.name:.001")}; + server.parse_msg(line1.get()); server.parse_msg(line2.get()); auto map = server.measurements(); EXPECT_DOUBLE_EQ(map["spectatord.parsedCount|statistic=count"], 2); - EXPECT_DOUBLE_EQ(map["dist.summary|statistic=count"], 2); - EXPECT_DOUBLE_EQ(map["dist.summary|statistic=totalAmount"], 3); - EXPECT_DOUBLE_EQ(map["dist.summary|statistic=max"], 2); + EXPECT_DOUBLE_EQ(map["timer.name|statistic=count"], 2); + EXPECT_DOUBLE_EQ(map["timer.name|statistic=totalTime"], 0.002); + EXPECT_DOUBLE_EQ(map["timer.name|statistic=max"], 0.001); } } // namespace diff --git a/spectator/CMakeLists.txt b/spectator/CMakeLists.txt index 1092b83..048fa4f 100644 --- a/spectator/CMakeLists.txt +++ b/spectator/CMakeLists.txt @@ -37,6 +37,8 @@ add_library(spectator OBJECT "meter.h" "monotonic_counter.cc" "monotonic_counter.h" + "monotonic_counter_uint.cc" + "monotonic_counter_uint.h" "monotonic_sampled.cc" "monotonic_sampled.h" "percentile_bucket_tags.inc" diff --git a/spectator/monotonic_counter.cc b/spectator/monotonic_counter.cc index 8bbdaf0..980adef 100644 --- a/spectator/monotonic_counter.cc +++ b/spectator/monotonic_counter.cc @@ -4,37 +4,28 @@ namespace spectator { static constexpr auto kNaN = std::numeric_limits::quiet_NaN(); -static constexpr auto kMax = std::numeric_limits::max(); MonotonicCounter::MonotonicCounter(Id id) noexcept - : Meter{std::move(id)}, init_(false), value_(0), prev_value_(0) {} + : Meter{std::move(id)}, value_(kNaN), prev_value_(kNaN) {} -void MonotonicCounter::Set(uint64_t amount) noexcept { +void MonotonicCounter::Set(double amount) noexcept { Update(); value_.store(amount, std::memory_order_relaxed); } auto MonotonicCounter::Delta() const noexcept -> double { - if (!init_.load(std::memory_order_relaxed)) return kNaN; - - auto prev = prev_value_.load(std::memory_order_relaxed); - auto curr = value_.load(std::memory_order_relaxed); - - if (curr < prev) { - return kMax - prev + curr + 1; - } else { - return curr - prev; - } + return value_.load(std::memory_order_relaxed) - + prev_value_.load(std::memory_order_relaxed); } void MonotonicCounter::Measure(Measurements* results) const noexcept { auto delta = Delta(); - prev_value_.store(value_.load(std::memory_order_relaxed), std::memory_order_relaxed); - init_.store(true, std::memory_order_relaxed); - + prev_value_.store(value_.load(std::memory_order_relaxed), + std::memory_order_relaxed); if (delta > 0) { if (!count_id_) { - count_id_ = std::make_unique(MeterId().WithDefaultStat(refs().count())); + count_id_ = + std::make_unique(MeterId().WithDefaultStat(refs().count())); } results->emplace_back(*count_id_, delta); } diff --git a/spectator/monotonic_counter.h b/spectator/monotonic_counter.h index 5d449c0..b43f11b 100644 --- a/spectator/monotonic_counter.h +++ b/spectator/monotonic_counter.h @@ -9,13 +9,12 @@ class MonotonicCounter : public Meter { explicit MonotonicCounter(Id id) noexcept; void Measure(Measurements* results) const noexcept; - void Set(uint64_t amount) noexcept; + void Set(double amount) noexcept; auto Delta() const noexcept -> double; private: mutable std::unique_ptr count_id_; - mutable std::atomic init_; - mutable std::atomic value_; - mutable std::atomic prev_value_; + mutable std::atomic value_; + mutable std::atomic prev_value_; }; } // namespace spectator diff --git a/spectator/monotonic_counter_test.cc b/spectator/monotonic_counter_test.cc index b2c5dea..7b66565 100644 --- a/spectator/monotonic_counter_test.cc +++ b/spectator/monotonic_counter_test.cc @@ -8,7 +8,7 @@ using spectator::Id; using spectator::refs; auto getMonotonicCounter(std::string_view name) -> spectator::MonotonicCounter { - return spectator::MonotonicCounter(spectator::Id::Of(name)); + return spectator::MonotonicCounter(Id::Of(name)); } TEST(MonotonicCounter, Init) { @@ -26,42 +26,34 @@ TEST(MonotonicCounter, Init) { EXPECT_DOUBLE_EQ(c.Delta(), 42.0); } -TEST(MonotonicCounter, Overflow) { - auto max = std::numeric_limits::max(); +TEST(MonotonicCounter, NegativeDelta) { auto c = getMonotonicCounter("neg"); EXPECT_TRUE(std::isnan(c.Delta())); - // initialize overflow condition spectator::Measurements ms; - c.Set(max - 5); + c.Set(100.0); c.Measure(&ms); EXPECT_TRUE(ms.empty()); - // trigger overflow condition - c.Set(max + 1); + c.Set(99.0); c.Measure(&ms); - EXPECT_EQ(ms.size(), 1); - auto id = c.MeterId().WithStat(refs().count()); - auto expected1 = spectator::Measurement{id, 6.0}; - EXPECT_EQ(expected1, ms.back()); + EXPECT_TRUE(ms.empty()); - // normal increment conditions - c.Set(max + 2); + c.Set(98.0); c.Measure(&ms); - EXPECT_EQ(ms.size(), 2); - auto expected2 = spectator::Measurement{id, 1.0}; - EXPECT_EQ(expected2, ms.back()); + EXPECT_TRUE(ms.empty()); - c.Set(5); + c.Set(100.0); c.Measure(&ms); - EXPECT_EQ(ms.size(), 3); - auto expected3 = spectator::Measurement{id, 4.0}; - EXPECT_EQ(expected3, ms.back()); + EXPECT_EQ(ms.size(), 1); + auto id = c.MeterId().WithStat(refs().count()); + auto expected = spectator::Measurement{id, 2.0}; + EXPECT_EQ(expected, ms.front()); } TEST(MonotonicCounter, Id) { auto c = getMonotonicCounter("id"); - auto id = spectator::Id("id", spectator::Tags{}); + auto id = Id("id", spectator::Tags{}); EXPECT_EQ(c.MeterId(), id); } diff --git a/spectator/monotonic_counter_uint.cc b/spectator/monotonic_counter_uint.cc new file mode 100644 index 0000000..62918e9 --- /dev/null +++ b/spectator/monotonic_counter_uint.cc @@ -0,0 +1,43 @@ +#include "monotonic_counter_uint.h" +#include "common_refs.h" + +namespace spectator { + +static constexpr auto kNaN = std::numeric_limits::quiet_NaN(); +static constexpr auto kMax = std::numeric_limits::max(); + +MonotonicCounterUint::MonotonicCounterUint(Id id) noexcept + : Meter{std::move(id)}, init_(false), value_(0), prev_value_(0) {} + +void MonotonicCounterUint::Set(uint64_t amount) noexcept { + Update(); + value_.store(amount, std::memory_order_relaxed); +} + +auto MonotonicCounterUint::Delta() const noexcept -> double { + if (!init_.load(std::memory_order_relaxed)) return kNaN; + + auto prev = prev_value_.load(std::memory_order_relaxed); + auto curr = value_.load(std::memory_order_relaxed); + + if (curr < prev) { + return kMax - prev + curr + 1; + } else { + return curr - prev; + } +} + +void MonotonicCounterUint::Measure(Measurements* results) const noexcept { + auto delta = Delta(); + prev_value_.store(value_.load(std::memory_order_relaxed), std::memory_order_relaxed); + init_.store(true, std::memory_order_relaxed); + + if (delta > 0) { + if (!count_id_) { + count_id_ = std::make_unique(MeterId().WithDefaultStat(refs().count())); + } + results->emplace_back(*count_id_, delta); + } +} + +} // namespace spectator diff --git a/spectator/monotonic_counter_uint.h b/spectator/monotonic_counter_uint.h new file mode 100644 index 0000000..64ca7ab --- /dev/null +++ b/spectator/monotonic_counter_uint.h @@ -0,0 +1,21 @@ +#pragma once + +#include "meter.h" +#include + +namespace spectator { +class MonotonicCounterUint : public Meter { + public: + explicit MonotonicCounterUint(Id id) noexcept; + void Measure(Measurements* results) const noexcept; + + void Set(uint64_t amount) noexcept; + auto Delta() const noexcept -> double; + + private: + mutable std::unique_ptr count_id_; + mutable std::atomic init_; + mutable std::atomic value_; + mutable std::atomic prev_value_; +}; +} // namespace spectator diff --git a/spectator/monotonic_counter_uint_test.cc b/spectator/monotonic_counter_uint_test.cc new file mode 100644 index 0000000..9ccc89f --- /dev/null +++ b/spectator/monotonic_counter_uint_test.cc @@ -0,0 +1,113 @@ +#include "common_refs.h" +#include "monotonic_counter_uint.h" +#include + +namespace { + +using spectator::Id; +using spectator::refs; + +auto getMonotonicCounterUint(std::string_view name) -> spectator::MonotonicCounterUint { + return spectator::MonotonicCounterUint(spectator::Id::Of(name)); +} + +TEST(MonotonicCounterUint, Init) { + auto c = getMonotonicCounterUint("foo"); + EXPECT_TRUE(std::isnan(c.Delta())); + + c.Set(42); + EXPECT_TRUE(std::isnan(c.Delta())); + + spectator::Measurements ms; + c.Measure(&ms); + ASSERT_TRUE(ms.empty()); + + c.Set(84); + EXPECT_DOUBLE_EQ(c.Delta(), 42.0); +} + +TEST(MonotonicCounterUint, Overflow) { + auto max = std::numeric_limits::max(); + auto c = getMonotonicCounterUint("overflow"); + EXPECT_TRUE(std::isnan(c.Delta())); + + // initialize overflow condition + spectator::Measurements ms; + c.Set(max - 5); + c.Measure(&ms); + EXPECT_TRUE(ms.empty()); + + // trigger overflow condition + c.Set(max + 1); + c.Measure(&ms); + EXPECT_EQ(ms.size(), 1); + auto id = c.MeterId().WithStat(refs().count()); + auto expected1 = spectator::Measurement{id, 6.0}; + EXPECT_EQ(expected1, ms.back()); + + // normal increment conditions + c.Set(max + 2); + c.Measure(&ms); + EXPECT_EQ(ms.size(), 2); + auto expected2 = spectator::Measurement{id, 1.0}; + EXPECT_EQ(expected2, ms.back()); + + c.Set(5); + c.Measure(&ms); + EXPECT_EQ(ms.size(), 3); + auto expected3 = spectator::Measurement{id, 4.0}; + EXPECT_EQ(expected3, ms.back()); +} + +TEST(MonotonicCounterUint, Id) { + auto c = getMonotonicCounterUint("id"); + auto id = spectator::Id("id", spectator::Tags{}); + EXPECT_EQ(c.MeterId(), id); +} + +TEST(MonotonicCounterUint, Measure) { + auto c = getMonotonicCounterUint("measure"); + c.Set(42); + spectator::Measurements measures; + c.Measure(&measures); + ASSERT_TRUE(measures.empty()); // initialize + + EXPECT_DOUBLE_EQ(c.Delta(), 0.0) + << "MonotonicCounters should reset their value after being measured"; + + c.Set(84.0); + c.Measure(&measures); + auto id = c.MeterId().WithStat(refs().count()); + std::vector expected({{id, 42.0}}); + EXPECT_EQ(expected, measures); + + measures.clear(); + c.Measure(&measures); + EXPECT_TRUE(measures.empty()) + << "MonotonicCounters should not report delta=0"; +} + +TEST(MonotonicCounterUint, DefaultStatistic) { + spectator::Measurements measures; + auto c = spectator::MonotonicCounterUint(Id::Of("foo", {{"statistic", "totalAmount"}})); + c.Set(42); + c.Measure(&measures); + c.Set(84.0); + c.Measure(&measures); + auto id = Id::Of("foo", {{"statistic", "totalAmount"}}); + std::vector expected({{id, 42.0}}); + EXPECT_EQ(expected, measures); +} + +TEST(MonotonicCounterUint, Update) { + auto counter = getMonotonicCounterUint("m"); + counter.Set(1); + + auto t1 = counter.Updated(); + auto t2 = counter.Updated(); + EXPECT_EQ(t1, t2); + usleep(1); + counter.Set(2); + EXPECT_TRUE(counter.Updated() > t1); +} +} // namespace diff --git a/spectator/monotonic_sampled.cc b/spectator/monotonic_sampled.cc index a08f1b0..1e1d5bf 100644 --- a/spectator/monotonic_sampled.cc +++ b/spectator/monotonic_sampled.cc @@ -3,42 +3,41 @@ namespace spectator { static constexpr auto kNaN = std::numeric_limits::quiet_NaN(); -static constexpr auto kMax = std::numeric_limits::max(); MonotonicSampled::MonotonicSampled(Id id) noexcept : Meter{std::move(id)}, - init_{false}, - value_{0}, - prev_value_{0}, + value_{kNaN}, + prev_value_{kNaN}, ts_{0}, prev_ts_{0} {} -void MonotonicSampled::Set(uint64_t amount, int64_t ts_nanos) noexcept { +void MonotonicSampled::Set(double amount, int64_t ts_nanos) noexcept { Update(); - absl::MutexLock lock(&mutex_); - // ignore out-of-order points, overflows are not expected for timestamps + absl::MutexLock lock(&mutex_); + // ignore out of order points, or wrap arounds if (ts_nanos < ts_) { return; } - - if (init_) { + // only update prev values at most once per reporting interval + if (std::isnan(prev_value_)) { prev_value_ = value_; prev_ts_ = ts_; } - value_ = amount; ts_ = ts_nanos; } void MonotonicSampled::Measure(Measurements* results) const noexcept { auto sampled_delta = SampledRate(); + if (std::isnan(sampled_delta)) { + return; + } { absl::MutexLock lock(&mutex_); prev_value_ = value_; prev_ts_ = ts_; - init_ = true; } if (sampled_delta > 0) { @@ -51,14 +50,8 @@ void MonotonicSampled::Measure(Measurements* results) const noexcept { auto MonotonicSampled::SampledRate() const noexcept -> double { absl::MutexLock lock(&mutex_); - if (!init_) return kNaN; auto delta_t = (ts_ - prev_ts_) / 1e9; - - if (value_ < prev_value_) { - return (kMax - prev_value_ + value_ + 1) / delta_t; - } else { - return (value_ - prev_value_) / delta_t; - } + return (value_ - prev_value_) / delta_t; } -} // namespace spectator \ No newline at end of file +} // namespace spectator diff --git a/spectator/monotonic_sampled.h b/spectator/monotonic_sampled.h index e75c342..5618108 100644 --- a/spectator/monotonic_sampled.h +++ b/spectator/monotonic_sampled.h @@ -10,16 +10,15 @@ class MonotonicSampled : public Meter { explicit MonotonicSampled(Id id) noexcept; void Measure(Measurements* results) const noexcept; - void Set(uint64_t amount, int64_t ts_nanos) noexcept; + void Set(double amount, int64_t ts_nanos) noexcept; auto SampledRate() const noexcept -> double; private: mutable std::unique_ptr count_id_; mutable absl::Mutex mutex_; - mutable bool init_ ABSL_GUARDED_BY(mutex_); - uint64_t value_ ABSL_GUARDED_BY(mutex_); - mutable uint64_t prev_value_ ABSL_GUARDED_BY(mutex_); - uint64_t ts_ ABSL_GUARDED_BY(mutex_); - mutable uint64_t prev_ts_ ABSL_GUARDED_BY(mutex_); + double value_ ABSL_GUARDED_BY(mutex_); + mutable double prev_value_ ABSL_GUARDED_BY(mutex_); + int64_t ts_ ABSL_GUARDED_BY(mutex_); + mutable int64_t prev_ts_ ABSL_GUARDED_BY(mutex_); }; } // namespace spectator diff --git a/spectator/monotonic_sampled_test.cc b/spectator/monotonic_sampled_test.cc index f7ef355..fc2a52f 100644 --- a/spectator/monotonic_sampled_test.cc +++ b/spectator/monotonic_sampled_test.cc @@ -3,10 +3,11 @@ namespace { +using spectator::Id; using spectator::refs; auto getMonotonicSampled(std::string_view name) { - return spectator::MonotonicSampled(spectator::Id::Of(name)); + return spectator::MonotonicSampled(Id::Of(name)); } auto s_to_ns(int seconds) { return int64_t(1000) * 1000 * 1000 * seconds; } @@ -26,42 +27,35 @@ TEST(MonotonicSampled, Init) { EXPECT_DOUBLE_EQ(c.SampledRate(), 42.0); } -TEST(MonotonicSampled, Overflow) { - auto max = std::numeric_limits::max(); +TEST(MonotonicSampled, NegativeDelta) { auto c = getMonotonicSampled("neg"); EXPECT_TRUE(std::isnan(c.SampledRate())); - // initialize overflow condition spectator::Measurements ms; - c.Set(max - 5, s_to_ns(1)); + c.Set(100.0, s_to_ns(1)); c.Measure(&ms); EXPECT_TRUE(ms.empty()); - // trigger overflow condition - c.Set(max + 1, s_to_ns(2)); + c.Set(99.0, s_to_ns(2)); c.Measure(&ms); - EXPECT_EQ(ms.size(), 1); - auto id = c.MeterId().WithStat(refs().count()); - auto expected1 = spectator::Measurement{id, 6.0}; - EXPECT_EQ(expected1, ms.back()); + EXPECT_TRUE(ms.empty()); - // normal increment conditions - c.Set(max + 2, s_to_ns(3)); + c.Set(98.0, s_to_ns(3)); c.Measure(&ms); - EXPECT_EQ(ms.size(), 2); - auto expected2 = spectator::Measurement{id, 1.0}; - EXPECT_EQ(expected2, ms.back()); + EXPECT_TRUE(ms.empty()); - c.Set(5, s_to_ns(4)); + c.Set(100.0, s_to_ns(4)); c.Measure(&ms); - EXPECT_EQ(ms.size(), 3); - auto expected3 = spectator::Measurement{id, 4.0}; - EXPECT_EQ(expected3, ms.back()); + ASSERT_EQ(ms.size(), 1); + auto id = c.MeterId().WithStat(refs().count()); + auto expected = spectator::Measurement{id, 2.0}; + EXPECT_EQ(expected, ms.front()); } + TEST(MonotonicSampled, Id) { auto c = getMonotonicSampled("id"); - auto id = spectator::Id::Of("id"); + auto id = Id::Of("id"); EXPECT_EQ(c.MeterId(), id); } diff --git a/spectator/registry.cc b/spectator/registry.cc index 3a927cd..ccddcf5 100644 --- a/spectator/registry.cc +++ b/spectator/registry.cc @@ -108,6 +108,16 @@ auto Registry::GetMonotonicCounter(std::string_view name, Tags tags) noexcept return GetMonotonicCounter(Id::Of(name, std::move(tags))); } +auto Registry::GetMonotonicCounterUint(Id id) noexcept + -> std::shared_ptr { + return all_meters_.insert_mono_counter_uint(std::move(id)); +} + +auto Registry::GetMonotonicCounterUint(std::string_view name, Tags tags) noexcept + -> std::shared_ptr { + return GetMonotonicCounterUint(Id::Of(name, std::move(tags))); +} + auto Registry::GetMonotonicSampled(Id id) noexcept -> std::shared_ptr { return all_meters_.insert_mono_sampled(std::move(id)); diff --git a/spectator/registry.h b/spectator/registry.h index e462814..40fa3ca 100644 --- a/spectator/registry.h +++ b/spectator/registry.h @@ -10,6 +10,7 @@ #include "gauge.h" #include "max_gauge.h" #include "monotonic_counter.h" +#include "monotonic_counter_uint.h" #include "publisher.h" #include "timer.h" #include "spectator/monotonic_sampled.h" @@ -158,13 +159,14 @@ struct all_meters { meter_map gauges_; meter_map max_gauges_; meter_map mono_counters_; + meter_map mono_counters_uint_; meter_map mono_sampled_; meter_map timers_; auto size() const -> size_t { return age_gauges_.size() + counters_.size() + dist_sums_.size() + gauges_.size() + max_gauges_.size() + mono_counters_.size() + - timers_.size(); + mono_counters_uint_.size() + timers_.size(); } auto measure(int64_t meter_ttl) const -> std::vector { @@ -176,6 +178,7 @@ struct all_meters { gauges_.measure(&res, meter_ttl); max_gauges_.measure(&res, meter_ttl); mono_counters_.measure(&res, meter_ttl); + mono_counters_uint_.measure(&res, meter_ttl); timers_.measure(&res, meter_ttl); return res; } @@ -203,6 +206,9 @@ struct all_meters { std::tie(expired, count) = mono_counters_.remove_expired(meter_ttl); total_expired += expired; total_count += count; + std::tie(expired, count) = mono_counters_uint_.remove_expired(meter_ttl); + total_expired += expired; + total_count += count; std::tie(expired, count) = timers_.remove_expired(meter_ttl); total_expired += expired; total_count += count; @@ -217,6 +223,11 @@ struct all_meters { return counters_.insert(std::make_shared(std::move(id))); } + auto insert_dist_sum(Id id) { + return dist_sums_.insert( + std::make_shared(std::move(id))); + } + auto insert_gauge(Id id, absl::Duration ttl) { return gauges_.insert(std::make_shared(std::move(id), ttl)); } @@ -230,16 +241,16 @@ struct all_meters { std::make_shared(std::move(id))); } + auto insert_mono_counter_uint(Id id) { + return mono_counters_uint_.insert( + std::make_shared(std::move(id))); + } + auto insert_mono_sampled(Id id) { return mono_sampled_.insert( std::make_shared(std::move(id))); } - auto insert_dist_sum(Id id) { - return dist_sums_.insert( - std::make_shared(std::move(id))); - } - auto insert_timer(Id id) { return timers_.insert(std::make_shared(std::move(id))); } @@ -264,37 +275,50 @@ class Registry { void OnMeasurements(measurements_callback fn) noexcept; - auto GetCounter(Id id) noexcept -> std::shared_ptr; - auto GetCounter(std::string_view name, Tags tags = {}) noexcept - -> std::shared_ptr; - - auto GetMonotonicCounter(Id id) noexcept -> std::shared_ptr; - auto GetMonotonicCounter(std::string_view name, Tags tags = {}) noexcept - -> std::shared_ptr; - - auto GetAgeGauge(Id id) noexcept -> std::shared_ptr; + auto GetAgeGauge(Id id) noexcept + -> std::shared_ptr; auto GetAgeGauge(std::string_view name, Tags tags = {}) noexcept -> std::shared_ptr; - auto GetMonotonicSampled(Id id) noexcept -> std::shared_ptr; - auto GetMonotonicSampled(std::string_view name, Tags tags = {}) noexcept - -> std::shared_ptr; + auto GetCounter(Id id) noexcept + -> std::shared_ptr; + auto GetCounter(std::string_view name, Tags tags = {}) noexcept + -> std::shared_ptr; auto GetDistributionSummary(Id id) noexcept -> std::shared_ptr; auto GetDistributionSummary(std::string_view name, Tags tags = {}) noexcept -> std::shared_ptr; - auto GetGauge(Id id) noexcept -> std::shared_ptr; - auto GetGauge(Id id, absl::Duration ttl) noexcept -> std::shared_ptr; + auto GetGauge(Id id) noexcept + -> std::shared_ptr; + auto GetGauge(Id id, absl::Duration ttl) noexcept + -> std::shared_ptr; auto GetGauge(std::string_view name, Tags tags = {}) noexcept -> std::shared_ptr; - auto GetMaxGauge(Id id) noexcept -> std::shared_ptr; + auto GetMaxGauge(Id id) noexcept + -> std::shared_ptr; auto GetMaxGauge(std::string_view name, Tags tags = {}) noexcept -> std::shared_ptr; - auto GetTimer(Id id) noexcept -> std::shared_ptr; + auto GetMonotonicCounter(Id id) noexcept + -> std::shared_ptr; + auto GetMonotonicCounter(std::string_view name, Tags tags = {}) noexcept + -> std::shared_ptr; + + auto GetMonotonicCounterUint(Id id) noexcept + -> std::shared_ptr; + auto GetMonotonicCounterUint(std::string_view name, Tags tags = {}) noexcept + -> std::shared_ptr; + + auto GetMonotonicSampled(Id id) noexcept + ->std::shared_ptr; + auto GetMonotonicSampled(std::string_view name, Tags tags = {}) noexcept + -> std::shared_ptr; + + auto GetTimer(Id id) noexcept + -> std::shared_ptr; auto GetTimer(std::string_view name, Tags tags = {}) noexcept -> std::shared_ptr; @@ -314,26 +338,29 @@ class Registry { void DeleteAllMeters(const std::string& type); // for debugging / testing - auto Timers() const -> std::vector { - return all_meters_.timers_.get_values(); + auto AgeGauges() const -> std::vector { + return all_meters_.age_gauges_.get_values(); } auto Counters() const -> std::vector { return all_meters_.counters_.get_values(); } - auto MonotonicCounters() const -> std::vector { - return all_meters_.mono_counters_.get_values(); + auto DistSummaries() const -> std::vector { + return all_meters_.dist_sums_.get_values(); } auto Gauges() const -> std::vector { return all_meters_.gauges_.get_values(); } - auto AgeGauges() const -> std::vector { - return all_meters_.age_gauges_.get_values(); - } auto MaxGauges() const -> std::vector { return all_meters_.max_gauges_.get_values(); } - auto DistSummaries() const -> std::vector { - return all_meters_.dist_sums_.get_values(); + auto MonotonicCounters() const -> std::vector { + return all_meters_.mono_counters_.get_values(); + } + auto MonotonicCountersUint() const -> std::vector { + return all_meters_.mono_counters_uint_.get_values(); + } + auto Timers() const -> std::vector { + return all_meters_.timers_.get_values(); } auto GetLastSuccessTime() const -> int64_t { return publisher_.GetLastSuccessTime();