From fbc82ffda5036dd017f55511ce487198d307fd7e Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Wed, 25 Dec 2024 16:56:19 +0300 Subject: [PATCH] feat: add metric send to Honeycomb --- .changeset/wicked-ties-know.md | 5 +++ packages/sync-service/config/runtime.exs | 7 ++++ packages/sync-service/lib/electric/config.ex | 3 +- .../lib/electric/plug/serve_shape_plug.ex | 5 ++- .../sync-service/lib/electric/telemetry.ex | 42 ++++++++++++------- packages/sync-service/mix.exs | 1 + packages/sync-service/mix.lock | 10 +++-- 7 files changed, 52 insertions(+), 21 deletions(-) create mode 100644 .changeset/wicked-ties-know.md diff --git a/.changeset/wicked-ties-know.md b/.changeset/wicked-ties-know.md new file mode 100644 index 0000000000..46ba997061 --- /dev/null +++ b/.changeset/wicked-ties-know.md @@ -0,0 +1,5 @@ +--- +"@core/sync-service": patch +--- + +Add metric publishing to Honeycomb when that exporter is enabled diff --git a/packages/sync-service/config/runtime.exs b/packages/sync-service/config/runtime.exs index 5ad3145eae..0a7a5ecd0b 100644 --- a/packages/sync-service/config/runtime.exs +++ b/packages/sync-service/config/runtime.exs @@ -74,6 +74,13 @@ if otlp_endpoint do otlp_endpoint: otlp_endpoint, otlp_headers: headers, otlp_compression: :gzip + + config :otel_metric_exporter, + otlp_protocol: :http_protobuf, + otlp_endpoint: otlp_endpoint, + otlp_headers: headers, + otlp_compression: :gzip, + resource: %{service: %{name: service_name, version: version}, instance: %{id: instance_id}} end otel_batch_processor = diff --git a/packages/sync-service/lib/electric/config.ex b/packages/sync-service/lib/electric/config.ex index e339cf781f..3ca837f22d 100644 --- a/packages/sync-service/lib/electric/config.ex +++ b/packages/sync-service/lib/electric/config.ex @@ -118,7 +118,8 @@ defmodule Electric.Config do def telemetry_export_enabled? do not is_nil(Electric.Config.get_env(:telemetry_statsd_host)) or not is_nil(Electric.Config.get_env(:prometheus_port)) or - Electric.Config.get_env(:call_home_telemetry?) + Electric.Config.get_env(:call_home_telemetry?) or + not is_nil(Application.get_env(:otel_metric_exporter, :otlp_endpoint)) end @doc ~S""" diff --git a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex index 620198c2ac..b89cafb319 100644 --- a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex +++ b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex @@ -599,7 +599,7 @@ defmodule Electric.Plug.ServeShapePlug do defp start_telemetry_span(conn, _) do OpentelemetryTelemetry.start_telemetry_span(OpenTelemetry, "Plug_shape_get", %{}, %{}) add_span_attrs_from_conn(conn) - conn + put_private(conn, :electric_telemetry_span, %{start_time: System.monotonic_time()}) end # Assign root span attributes based on the latest state of Plug.Conn and end the root span. @@ -613,7 +613,8 @@ defmodule Electric.Plug.ServeShapePlug do %{ count: 1, bytes: assigns[:streaming_bytes_sent] || 0, - monotonic_time: System.monotonic_time() + monotonic_time: System.monotonic_time(), + duration: System.monotonic_time() - conn.private[:electric_telemetry_span][:start_time] }, %{ live: assigns[:live], diff --git a/packages/sync-service/lib/electric/telemetry.ex b/packages/sync-service/lib/electric/telemetry.ex index 89e4bc27d3..e40a2a66bc 100644 --- a/packages/sync-service/lib/electric/telemetry.ex +++ b/packages/sync-service/lib/electric/telemetry.ex @@ -20,6 +20,7 @@ defmodule Electric.Telemetry do statsd_host = Electric.Config.get_env(:telemetry_statsd_host) prometheus? = not is_nil(Electric.Config.get_env(:prometheus_port)) call_home_telemetry? = Electric.Config.get_env(:call_home_telemetry?) + otel_metrics? = not is_nil(Application.get_env(:otel_metric_exporter, :otlp_endpoint)) [ {:telemetry_poller, @@ -28,12 +29,17 @@ defmodule Electric.Telemetry do init_delay: :timer.seconds(5)}, statsd_reporter_child_spec(statsd_host), prometheus_reporter_child_spec(prometheus?), - call_home_reporter_child_spec(call_home_telemetry?) + call_home_reporter_child_spec(call_home_telemetry?), + otel_reporter_child_spec(otel_metrics?) ] |> Enum.reject(&is_nil/1) |> Supervisor.init(strategy: :one_for_one) end + defp otel_reporter_child_spec(true) do + {OtelMetricExporter, metrics: otel_metrics(), export_period: :timer.seconds(30)} + end + defp call_home_reporter_child_spec(false), do: nil defp call_home_reporter_child_spec(true) do @@ -193,22 +199,30 @@ defmodule Electric.Telemetry do last_value("vm.total_run_queue_lengths.total"), last_value("vm.total_run_queue_lengths.cpu"), last_value("vm.total_run_queue_lengths.io"), - last_value("electric.postgres.replication.wal_size", unit: :byte) - # distribution("plug.router_dispatch.stop.duration", - # tags: [:route], - # unit: {:native, :millisecond} - # ), - # distribution("plug.router_dispatch.exception.duration", - # tags: [:route], - # unit: {:native, :millisecond} - # ), - # distribution("electric.query.duration", unit: {:native, :millisecond}), - # distribution("electric.query.serialization_duration", unit: {:native, :millisecond}), - # distribution("electric.snapshot.storage", unit: {:native, :millisecond}), - # distribution("electric.snapshot.encoding", unit: {:native, :millisecond}) + last_value("electric.postgres.replication.wal_size", unit: :byte), + counter("electric.postgres.replication.transaction_received.count"), + sum("electric.postgres.replication.transaction_received.bytes", unit: :byte), + sum("electric.storage.transaction_stored.bytes", unit: :byte) ] end + defp otel_metrics() do + [ + last_value("system.load_percent.avg1"), + last_value("system.load_percent.avg5"), + last_value("system.load_percent.avg15"), + distribution("electric.plug.serve_shape.duration", + drop: &(Map.get(&1, :live, false) || false) + ), + distribution("electric.shape_cache.create_snapshot_task.stop.duration", + unit: {:native, :millisecond} + ), + distribution("electric.storage.make_new_snapshot.stop.duration", + unit: {:native, :millisecond} + ) + ] ++ prometheus_metrics() + end + defp periodic_measurements(opts) do stack_id = Keyword.fetch!(opts, :stack_id) diff --git a/packages/sync-service/mix.exs b/packages/sync-service/mix.exs index 6f2803bdea..50d0c1883c 100644 --- a/packages/sync-service/mix.exs +++ b/packages/sync-service/mix.exs @@ -86,6 +86,7 @@ defmodule Electric.MixProject do {:opentelemetry_exporter, "~> 1.8"}, {:opentelemetry_telemetry, "~> 1.1"}, {:opentelemetry_semantic_conventions, "~> 1.27"}, + {:otel_metric_exporter, "~> 0.2"}, {:pg_query_ex, "0.5.3"}, {:plug, "~> 1.16"}, {:postgrex, "~> 0.19"}, diff --git a/packages/sync-service/mix.lock b/packages/sync-service/mix.lock index 6c8b70a500..d801235f33 100644 --- a/packages/sync-service/mix.lock +++ b/packages/sync-service/mix.lock @@ -7,7 +7,7 @@ "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, "cubdb": {:hex, :cubdb, "2.0.2", "d4253885084dae37a8ff73887d232864eb38ecac962aa08543e686b0183a1d62", [:mix], [], "hexpm", "c99cc8f9e6c4deb98d16cca5ded1928edd22e48b4736b76e8a1a85367d7fe921"}, "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, - "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "dotenvy": {:hex, :dotenvy, "0.8.0", "777486ad485668317c56afc53a7cbcd74f43e4e34588ba8e95a73e15a360050e", [:mix], [], "hexpm", "1f535066282388cbd109743d337ac46ff0708195780d4b5778bb83491ab1b654"}, "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, @@ -38,20 +38,22 @@ "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.8.0", "5d546123230771ef4174e37bedfd77e3374913304cd6ea3ca82a2add49cd5d56", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.5.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "a1f9f271f8d3b02b81462a6bfef7075fd8457fdb06adff5d2537df5e2264d9af"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "1.27.0", "acd0194a94a1e57d63da982ee9f4a9f88834ae0b31b0bd850815fe9be4bbb45f", [:mix, :rebar3], [], "hexpm", "9681ccaa24fd3d810b4461581717661fd85ff7019b082c2dff89c7d5b1fc2864"}, "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.1.1", "4a73bfa29d7780ffe33db345465919cef875034854649c37ac789eb8e8f38b21", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ee43b14e6866123a3ee1344e3c0d3d7591f4537542c2a925fcdbf46249c9b50b"}, + "otel_metric_exporter": {:hex, :otel_metric_exporter, "0.2.0", "8ce04170fc8ceb435439850e9eaf83a1aeb1e6d7e3bd22440da067316a1514dc", [:mix], [{:finch, "~> 0.19", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.13.0", [hex: :protobuf, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "419a624567bf59f1ae7b10d25e8fc31864a29932aac72f0ebcea072a5331360c"}, "pg_query_ex": {:hex, :pg_query_ex, "0.5.3", "84bf09f4ea10ada6e1cbfd739ccb9ffb6e5b21d87ab81cf52a42fefcc1808566", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:protox, "~> 1.7", [hex: :protox, repo: "hexpm", optional: false]}], "hexpm", "ec0554d6d287da4cc15cc773577ef61cf41d5d6fcc784bb85f6209439cb246a7"}, "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "postgrex": {:hex, :postgrex, "0.19.0", "f7d50e50cb42e0a185f5b9a6095125a9ab7e4abccfbe2ab820ab9aa92b71dbab", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "dba2d2a0a8637defbf2307e8629cb2526388ba7348f67d04ec77a5d6a72ecfae"}, + "protobuf": {:hex, :protobuf, "0.13.0", "7a9d9aeb039f68a81717eb2efd6928fdf44f03d2c0dfdcedc7b560f5f5aae93d", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "21092a223e3c6c144c1a291ab082a7ead32821ba77073b72c68515aa51fef570"}, "protox": {:hex, :protox, "1.7.3", "dff5488a648850c95cbd1cca5430be7ccedc99e4102aa934dbf60abfa30e64c1", [:mix], [{:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "b936c0654b68b306c4be853db23bb5623e2be89e11238908f2ff6da10fc0275f"}, "remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"}, "req": {:hex, :req, "0.5.7", "b722680e03d531a2947282adff474362a48a02aa54b131196fbf7acaff5e4cee", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "c6035374615120a8923e8089d0c21a3496cf9eda2d287b806081b8f323ceee29"}, "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"}, "sentry": {:hex, :sentry, "10.8.0", "1e8cc0ef21401e5914e6fc2f37489d6c685d31a0556dbd8ab4709cc1587a7232", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0 or ~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "92549e7ba776b7ccfed4e74d58987272d37d99606b130e4141bc015a1a8e4235"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, - "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, - "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"}, + "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, + "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"}, "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"}, - "telemetry_metrics_statsd": {:hex, :telemetry_metrics_statsd, "0.7.0", "92732fae63db31ef2508df6faee7d81401883e33f2976715a82f296a33a45cee", [:mix], [{:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "797e34a856376dfd4e96347da0f747fcff4e0cadf6e6f0f989598f563cad05ff"}, + "telemetry_metrics_statsd": {:hex, :telemetry_metrics_statsd, "0.7.1", "3502235bb5b35ce50d608bf0f34369ef76eb92a4dbc8708c7e8780ca0da2d53e", [:mix], [{:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "06338d9dc3b4a202f11a6e706fd3feba4c46100d0aca23688dea0b8f801c361f"}, "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"}, "tls_certificate_check": {:hex, :tls_certificate_check, "1.24.0", "d00e2887551ff8cdae4d0340d90d9fcbc4943c7b5f49d32ed4bc23aff4db9a44", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "90b25a58ee433d91c17f036d4d354bf8859a089bfda60e68a86f8eecae45ef1b"},