From aba564b45132b8e1a9593379224957ee88335b77 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Wed, 18 Dec 2024 16:18:16 +0100 Subject: [PATCH 1/2] fix(tracing): Return empty receivers list if tracing backend isn't connected --- src/grafana_agent.py | 13 +++++++- tests/scenario/test_tracing_integration.py | 35 ++++++++++++++++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/grafana_agent.py b/src/grafana_agent.py index 2758658..1725e26 100644 --- a/src/grafana_agent.py +++ b/src/grafana_agent.py @@ -288,10 +288,13 @@ def _requested_tracing_protocols(self) -> Set[ReceiverProtocol]: ) def _update_tracing_provider(self): + requested_tracing_protocols = ( + self._requested_tracing_protocols if self._tracing.is_ready() else [] + ) self._tracing_provider.publish_receivers( tuple( (protocol, self._get_tracing_receiver_url(protocol)) - for protocol in self._requested_tracing_protocols + for protocol in requested_tracing_protocols ) ) @@ -880,6 +883,14 @@ def _tracing_receivers(self) -> Dict[str, Union[Any, List[Any]]]: Returns: a dict with the receivers config. """ + if not self._tracing.is_ready(): + # we need to guard against the "upstream" tracing backend relation as grafana-agent fails to start + # if we don't provide the tracing endpoint to send data to. + logger.warning( + "Tracing backend is not connected yet: grafana-agent cannot ingest traces." + ) + return {} + receivers_set = self._requested_tracing_protocols if not receivers_set: diff --git a/tests/scenario/test_tracing_integration.py b/tests/scenario/test_tracing_integration.py index 6175631..432b346 100644 --- a/tests/scenario/test_tracing_integration.py +++ b/tests/scenario/test_tracing_integration.py @@ -39,6 +39,37 @@ def base_state(): def test_tracing_relation(ctx, base_state): + # GIVEN a tracing relation over the tracing-provider endpoint + tracing_provider = Relation( + "tracing-provider", + remote_app_data=TracingRequirerAppData(receivers=["otlp_http", "otlp_grpc"]).dump(), + ) + tracing = Relation( + "tracing", + remote_app_data=TracingProviderAppData( + receivers=[ + Receiver(protocol={"name": "otlp_grpc", "type": "grpc"}, url="http:foo.com:1111") + ] + ).dump(), + ) + + state = dataclasses.replace(base_state, relations=[tracing, tracing_provider]) + # WHEN we process any setup event for the relation + state_out = ctx.run(ctx.on.relation_changed(tracing_provider), state) + + agent = state_out.get_container("agent") + + # THEN the agent has started + assert agent.services["agent"].is_running() + # AND the grafana agent config has a traces config section + fs = agent.get_filesystem(ctx) + gagent_config = fs.joinpath(*CONFIG_PATH.strip("/").split("/")) + assert gagent_config.exists() + yml = yaml.safe_load(gagent_config.read_text()) + assert yml["traces"]["configs"][0], yml.get("traces", "") + + +def test_tracing_provider_without_tracing(ctx, base_state): # GIVEN a tracing relation over the tracing-provider endpoint tracing = Relation( "tracing-provider", @@ -53,12 +84,12 @@ def test_tracing_relation(ctx, base_state): # THEN the agent has started assert agent.services["agent"].is_running() - # AND the grafana agent config has a traces config section + # AND the grafana agent config has an empty traces config section fs = agent.get_filesystem(ctx) gagent_config = fs.joinpath(*CONFIG_PATH.strip("/").split("/")) assert gagent_config.exists() yml = yaml.safe_load(gagent_config.read_text()) - assert yml["traces"]["configs"][0], yml.get("traces", "") + assert yml["traces"] == {} def test_tracing_relations_in_and_out(ctx, base_state): From 3220d906dd29edb29ccbef2f79dc3da213aba9e0 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Thu, 19 Dec 2024 09:23:25 +0100 Subject: [PATCH 2/2] Comment on tracing config limitations --- src/grafana_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/grafana_agent.py b/src/grafana_agent.py index 1725e26..091023d 100644 --- a/src/grafana_agent.py +++ b/src/grafana_agent.py @@ -288,6 +288,8 @@ def _requested_tracing_protocols(self) -> Set[ReceiverProtocol]: ) def _update_tracing_provider(self): + # If the "upstream" tracing is not ready, we don't want to publish the receivers. + # Otherwise, charms that integrate over `tracing` would start sending traces to an endpoint that isn't open. requested_tracing_protocols = ( self._requested_tracing_protocols if self._tracing.is_ready() else [] ) @@ -884,8 +886,6 @@ def _tracing_receivers(self) -> Dict[str, Union[Any, List[Any]]]: a dict with the receivers config. """ if not self._tracing.is_ready(): - # we need to guard against the "upstream" tracing backend relation as grafana-agent fails to start - # if we don't provide the tracing endpoint to send data to. logger.warning( "Tracing backend is not connected yet: grafana-agent cannot ingest traces." )