Sample grpc and block metrics exported and tested

helium · May 10, 2022 · 35c3f3e · 35c3f3e
1 parent ba9de18
commit 35c3f3e
Show file tree

Hide file tree

Showing 6 changed files with 152 additions and 355 deletions.
diff --git a/include/metrics.hrl b/include/metrics.hrl
@@ -2,10 +2,20 @@
 
 -define(METRICS_BLOCK_ABSORB, "blockchain_block_absorb_duration").
 -define(METRICS_BLOCK_HEIGHT, "blockchain_block_height").
+-define(METRICS_GRPC_SESSIONS, "grpcbox_session_count").
+-define(METRICS_GRPC_LATENCY, "grpcbox_session_latency").
 
 -define(METRICS, #{
-    block_metrics => [
-        {?METRICS_BLOCK_ABSORB, [blockchain, block, absorb], prometheus_histogram, [stage], "Block absorb duration"},
-        {?METRICS_BLOCK_HEIGHT, [blockchain, block, height], prometheus_gauge, [time], "Most recent block height"}
-    ]
+    block_metrics => {
+        [ [blockchain, block, absorb],
+          [blockchain, block, height] ],
+        [ {?METRICS_BLOCK_ABSORB, prometheus_histogram, [stage], "Block absorb duration"},
+          {?METRICS_BLOCK_HEIGHT, prometheus_gauge, [time], "Most recent block height"} ]
+    },
+    grpc_metrics => {
+        [ [grpcbox, server, rpc_begin],
+          [grpcbox, server, rpc_end] ],
+        [ {?METRICS_GRPC_SESSIONS, prometheus_gauge, [method], "Grpc session count"},
+          {?METRICS_GRPC_LATENCY, prometheus_histogram, [method, status], "Grpc session latency"} ]
+    }
 }).
diff --git a/rebar.config b/rebar.config
@@ -21,6 +21,7 @@
         {jsx, "3.1.0"},
         {kvc, {git, "https://github.com/etrepum/kvc", {tag, "v1.7.0"}}},
         {longfi, {git, "https://github.com/helium/longfi-erlang", {tag, "0.2.2"}}},
+        {grpcbox, {git, "https://github.com/jeffgrunewald/grpcbox.git", {branch, "jg/add_telemetry_handler"}}},
         {grpc_lib, {git, "https://github.com/Bluehouse-Technology/grpc_lib", {branch, "master"}}},
         {grpc_client, {git, "https://github.com/Bluehouse-Technology/grpc_client.git", {branch, "master"}}},
         {http2_client, {git, "https://github.com/Bluehouse-Technology/http2_client",

diff --git a/rebar.lock b/rebar.lock
@@ -1,5 +1,5 @@
 {"1.2.0",
-[{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},2},
+[{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},1},
  {<<"backoff">>,{pkg,<<"backoff">>,<<"1.1.6">>},2},
  {<<"base32">>,{pkg,<<"base32">>,<<"0.1.0">>},4},
  {<<"base64url">>,{pkg,<<"base64url">>,<<"1.0.1">>},1},
@@ -10,8 +10,8 @@
  {<<"certifi">>,{pkg,<<"certifi">>,<<"2.8.0">>},2},
  {<<"chatterbox">>,
   {git,"https://github.com/andymck/chatterbox",
-       {ref,"b56a2a08b8a31b7ab0d536e5e8a55d5abe70390f"}},
-  2},
+       {ref,"fe0a96049723ac07174cb08ed6d8a1b4f6997be9"}},
+  1},
  {<<"clique">>,
   {git,"https://github.com/helium/clique.git",
        {ref,"e4be0dae150061bec080502ed530091672880867"}},
@@ -20,7 +20,7 @@
   {git,"https://github.com/ninenines/cowlib",
        {ref,"ce6798c6b2e95b6a34c6a76d2489eaf159827d80"}},
   1},
- {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},2},
+ {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},1},
  {<<"cuttlefish">>,
   {git,"https://github.com/helium/cuttlefish.git",
        {ref,"8672838e8f4ef61602aee6e4ff97ec9be54031dc"}},
@@ -83,7 +83,7 @@
  {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},3},
  {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},2},
  {<<"gpb">>,{pkg,<<"gpb">>,<<"4.19.5">>},1},
- {<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},2},
+ {<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},1},
  {<<"grpc_client">>,
   {git,"https://github.com/Bluehouse-Technology/grpc_client.git",
        {ref,"02689e7992ef70b42b6b890db450ae4a7c21a50c"}},
@@ -93,9 +93,9 @@
        {ref,"a77686b55b60b052d6c7cd927d04dde429bbdabf"}},
   0},
  {<<"grpcbox">>,
-  {git,"https://github.com/andymck/grpcbox.git",
-       {ref,"fbf689bb9c25fc2943155c891974e1f745ce5ac7"}},
-  1},
+  {git,"https://github.com/jeffgrunewald/grpcbox.git",
+       {ref,"2bb831dbf3b9050aa34ec0d88a36d32125c41a52"}},
+  0},
  {<<"h3">>,
   {git,"https://github.com/helium/erlang-h3.git",
        {ref,"90e1b6ebf93f88702ce8d24d9142833a8401e3ab"}},
@@ -109,7 +109,7 @@
   {git,"https://github.com/helium/proto.git",
        {ref,"73ed90cc875451ddcdff968a368c586e55eb730a"}},
   1},
- {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},3},
+ {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},2},
  {<<"http2_client">>,
   {git,"https://github.com/Bluehouse-Technology/http2_client",
        {ref,"8a6faf3ef1e0f3bb480d52d34ef5181258e99302"}},

diff --git a/src/metrics/miner_metrics_server.erl b/src/metrics/miner_metrics_server.erl
@@ -49,8 +49,8 @@ start_link() ->
 
 init(_Args) ->
     case get_configs() of
-        [] -> ignore;
-        Metrics ->
+        {[], []} -> ignore;
+        {[_ | _], [_ | _]} = Metrics ->
             erlang:process_flag(trap_exit, true),
 
             ok = setup_metrics(Metrics),
@@ -95,41 +95,33 @@ terminate(Reason, #state{metrics = Metrics, reporter_pid = Reporter}) ->
         Metrics
     ).
 
-setup_metrics(Metrics) ->
-    lager:warning("METRICS ~p", [Metrics]),
+setup_metrics({EventNames, EventSpecs}) ->
+    lager:warning("METRICS ~p", [EventSpecs]),
     lists:foreach(
-        fun({Metric, Event, Module, Meta, Description}) ->
+        fun({Metric, Module, Meta, Description}) ->
             lager:info("Declaring metric ~p as ~p meta=~p", [Metric, Module, Meta]),
+            MetricOpts = [{name, Metric}, {help, Description}, {labels, Meta}],
             case Module of
                 prometheus_histogram ->
-                    Module:declare([
-                        {name, Metric},
-                        {help, Description},
-                        {labels, Meta},
-                        {buckets, ?METRICS_HISTOGRAM_BUCKETS}
-                    ]);
+                    Module:declare(MetricOpts ++ [{buckets, ?METRICS_HISTOGRAM_BUCKETS}]);
                 _ ->
-                    Module:declare([
-                        {name, Metric},
-                        {help, Description},
-                        {labels, Meta}
-                    ])
-            end,
-
-            ok = telemetry:attach(list_to_binary(Metric), Event, fun miner_metrics_server:handle_metric/4, [])
+                    Module:declare(MetricOpts)
+            end
         end,
-        Metrics
-    ).
+        EventSpecs
+    ),
+
+    ok = telemetry:attach_many(<<"miner-metrics-handler">>, EventNames, fun miner_metrics_server:handle_metric/4, []).
 
 get_configs() ->
     lists:foldl(
-        fun(Metric, Acc) ->
+        fun(Metric, {Names, Specs} = Acc) ->
             case maps:get(Metric, ?METRICS, undefined) of
                 undefined -> Acc;
-                Result -> Acc ++ Result
+                {N, S} -> {Names ++ N, Specs ++ S}
             end
         end,
-        [],
+        {[], []},
         application:get_env(miner, metrics, [])
     ).
 
@@ -138,4 +130,12 @@ handle_metric_event([blockchain, block, absorb], #{duration := Duration}, #{stag
     ok;
 handle_metric_event([blockchain, block, height], #{height := Height}, #{time := Time}) ->
     prometheus_gauge:set(?METRICS_BLOCK_HEIGHT, [Time], Height),
+    ok;
+handle_metric_event([grpcbox, server, rpc_end], #{server_latency := Latency},
+                                                #{grpc_server_method := Method, grpc_server_status := Status}) ->
+    prometheus_gauge:dec(?METRICS_GRPC_SESSIONS, [Method]),
+    prometheus_histogram:observe(?METRICS_GRPC_LATENCY, [Method, Status], Latency),
+    ok;
+handle_metric_event([grpcbox, server, rpc_begin], _Measurements, #{grpc_server_method := Method}) ->
+    prometheus_gauge:inc(?METRICS_GRPC_SESSIONS, [Method]),
     ok.
diff --git a/test/miner_ct_utils.erl b/test/miner_ct_utils.erl
@@ -947,8 +947,9 @@ init_per_testcase(Mod, TestCase, Config0) ->
 
     GRPCServerConfigFun = fun(PeerPort)->
          [#{grpc_opts => #{service_protos => [gateway_pb],
-                           services => #{'helium.gateway' => helium_gateway_service}
-                            },
+                           services => #{'helium.gateway' => helium_gateway_service},
+                           stats_handler => grpcbox_telemetry_stats_handler
+                         },
 
             transport_opts => #{ssl => false},