diff --git a/config.yaml b/config.yaml index fedfc9b9c..1eb6ba8a9 100644 --- a/config.yaml +++ b/config.yaml @@ -16,6 +16,11 @@ options: The port on which the pgbouncer prometheus exporter serves metrics. type: int + vip: + description: | + Virtual IP to use to front pgbouncer units. Used only in case of external node connection. + type: string + pool_mode: default: session description: | diff --git a/lib/charms/grafana_agent/v0/cos_agent.py b/lib/charms/grafana_agent/v0/cos_agent.py index 870ba62a1..582b70c07 100644 --- a/lib/charms/grafana_agent/v0/cos_agent.py +++ b/lib/charms/grafana_agent/v0/cos_agent.py @@ -206,19 +206,34 @@ def __init__(self, *args): ``` """ +import enum import json import logging +import socket from collections import namedtuple from itertools import chain from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional, Set, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Callable, + ClassVar, + Dict, + List, + Literal, + MutableMapping, + Optional, + Set, + Tuple, + Union, +) import pydantic from cosl import GrafanaDashboard, JujuTopology from cosl.rules import AlertRules from ops.charm import RelationChangedEvent from ops.framework import EventBase, EventSource, Object, ObjectEvents -from ops.model import Relation +from ops.model import ModelError, Relation from ops.testing import CharmType if TYPE_CHECKING: @@ -234,9 +249,9 @@ class _MetricsEndpointDict(TypedDict): LIBID = "dc15fa84cef84ce58155fb84f6c6213a" LIBAPI = 0 -LIBPATCH = 8 +LIBPATCH = 10 -PYDEPS = ["cosl", "pydantic < 2"] +PYDEPS = ["cosl", "pydantic"] DEFAULT_RELATION_NAME = "cos-agent" DEFAULT_PEER_RELATION_NAME = "peers" @@ -249,7 +264,207 @@ class _MetricsEndpointDict(TypedDict): SnapEndpoint = namedtuple("SnapEndpoint", "owner, name") -class CosAgentProviderUnitData(pydantic.BaseModel): +# Note: MutableMapping is imported from the typing module and not collections.abc +# because subscripting collections.abc.MutableMapping was added in python 3.9, but +# most of our charms are based on 20.04, which has python 3.8. + +_RawDatabag = MutableMapping[str, str] + + +class TransportProtocolType(str, enum.Enum): + """Receiver Type.""" + + http = "http" + grpc = "grpc" + + +receiver_protocol_to_transport_protocol = { + "zipkin": TransportProtocolType.http, + "kafka": TransportProtocolType.http, + "tempo_http": TransportProtocolType.http, + "tempo_grpc": TransportProtocolType.grpc, + "otlp_grpc": TransportProtocolType.grpc, + "otlp_http": TransportProtocolType.http, + "jaeger_thrift_http": TransportProtocolType.http, +} + +_tracing_receivers_ports = { + # OTLP receiver: see + # https://github.com/open-telemetry/opentelemetry-collector/tree/v0.96.0/receiver/otlpreceiver + "otlp_http": 4318, + "otlp_grpc": 4317, + # Jaeger receiver: see + # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.96.0/receiver/jaegerreceiver + "jaeger_grpc": 14250, + "jaeger_thrift_http": 14268, + # Zipkin receiver: see + # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.96.0/receiver/zipkinreceiver + "zipkin": 9411, +} + +ReceiverProtocol = Literal["otlp_grpc", "otlp_http", "zipkin", "jaeger_thrift_http", "jaeger_grpc"] + + +class TracingError(Exception): + """Base class for custom errors raised by tracing.""" + + +class NotReadyError(TracingError): + """Raised by the provider wrapper if a requirer hasn't published the required data (yet).""" + + +class ProtocolNotRequestedError(TracingError): + """Raised if the user attempts to obtain an endpoint for a protocol it did not request.""" + + +class DataValidationError(TracingError): + """Raised when data validation fails on IPU relation data.""" + + +class AmbiguousRelationUsageError(TracingError): + """Raised when one wrongly assumes that there can only be one relation on an endpoint.""" + + +# TODO we want to eventually use `DatabagModel` from cosl but it likely needs a move to common package first +if int(pydantic.version.VERSION.split(".")[0]) < 2: # type: ignore + + class DatabagModel(pydantic.BaseModel): # type: ignore + """Base databag model.""" + + class Config: + """Pydantic config.""" + + # ignore any extra fields in the databag + extra = "ignore" + """Ignore any extra fields in the databag.""" + allow_population_by_field_name = True + """Allow instantiating this class by field name (instead of forcing alias).""" + + _NEST_UNDER = None + + @classmethod + def load(cls, databag: MutableMapping): + """Load this model from a Juju databag.""" + if cls._NEST_UNDER: + return cls.parse_obj(json.loads(databag[cls._NEST_UNDER])) + + try: + data = { + k: json.loads(v) + for k, v in databag.items() + # Don't attempt to parse model-external values + if k in {f.alias for f in cls.__fields__.values()} + } + except json.JSONDecodeError as e: + msg = f"invalid databag contents: expecting json. {databag}" + logger.error(msg) + raise DataValidationError(msg) from e + + try: + return cls.parse_raw(json.dumps(data)) # type: ignore + except pydantic.ValidationError as e: + msg = f"failed to validate databag: {databag}" + logger.debug(msg, exc_info=True) + raise DataValidationError(msg) from e + + def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): + """Write the contents of this model to Juju databag. + + :param databag: the databag to write the data to. + :param clear: ensure the databag is cleared before writing it. + """ + if clear and databag: + databag.clear() + + if databag is None: + databag = {} + + if self._NEST_UNDER: + databag[self._NEST_UNDER] = self.json(by_alias=True) + return databag + + dct = self.dict() + for key, field in self.__fields__.items(): # type: ignore + value = dct[key] + databag[field.alias or key] = json.dumps(value) + + return databag + +else: + from pydantic import ConfigDict + + class DatabagModel(pydantic.BaseModel): + """Base databag model.""" + + model_config = ConfigDict( + # ignore any extra fields in the databag + extra="ignore", + # Allow instantiating this class by field name (instead of forcing alias). + populate_by_name=True, + # Custom config key: whether to nest the whole datastructure (as json) + # under a field or spread it out at the toplevel. + _NEST_UNDER=None, # type: ignore + arbitrary_types_allowed=True, + ) + """Pydantic config.""" + + @classmethod + def load(cls, databag: MutableMapping): + """Load this model from a Juju databag.""" + nest_under = cls.model_config.get("_NEST_UNDER") # type: ignore + if nest_under: + return cls.model_validate(json.loads(databag[nest_under])) # type: ignore + + try: + data = { + k: json.loads(v) + for k, v in databag.items() + # Don't attempt to parse model-external values + if k in {(f.alias or n) for n, f in cls.__fields__.items()} + } + except json.JSONDecodeError as e: + msg = f"invalid databag contents: expecting json. {databag}" + logger.error(msg) + raise DataValidationError(msg) from e + + try: + return cls.model_validate_json(json.dumps(data)) # type: ignore + except pydantic.ValidationError as e: + msg = f"failed to validate databag: {databag}" + logger.debug(msg, exc_info=True) + raise DataValidationError(msg) from e + + def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): + """Write the contents of this model to Juju databag. + + :param databag: the databag to write the data to. + :param clear: ensure the databag is cleared before writing it. + """ + if clear and databag: + databag.clear() + + if databag is None: + databag = {} + nest_under = self.model_config.get("_NEST_UNDER") + if nest_under: + databag[nest_under] = self.model_dump_json( # type: ignore + by_alias=True, + # skip keys whose values are default + exclude_defaults=True, + ) + return databag + + dct = self.model_dump() # type: ignore + for key, field in self.model_fields.items(): # type: ignore + value = dct[key] + if value == field.default: + continue + databag[field.alias or key] = json.dumps(value) + + return databag + + +class CosAgentProviderUnitData(DatabagModel): """Unit databag model for `cos-agent` relation.""" # The following entries are the same for all units of the same principal. @@ -267,13 +482,16 @@ class CosAgentProviderUnitData(pydantic.BaseModel): metrics_scrape_jobs: List[Dict] log_slots: List[str] + # Requested tracing protocols. + tracing_protocols: Optional[List[str]] = None + # when this whole datastructure is dumped into a databag, it will be nested under this key. # while not strictly necessary (we could have it 'flattened out' into the databag), # this simplifies working with the model. KEY: ClassVar[str] = "config" -class CosAgentPeersUnitData(pydantic.BaseModel): +class CosAgentPeersUnitData(DatabagModel): """Unit databag model for `peers` cos-agent machine charm peer relation.""" # We need the principal unit name and relation metadata to be able to render identifiers @@ -304,6 +522,83 @@ def app_name(self) -> str: return self.unit_name.split("/")[0] +if int(pydantic.version.VERSION.split(".")[0]) < 2: # type: ignore + + class ProtocolType(pydantic.BaseModel): # type: ignore + """Protocol Type.""" + + class Config: + """Pydantic config.""" + + use_enum_values = True + """Allow serializing enum values.""" + + name: str = pydantic.Field( + ..., + description="Receiver protocol name. What protocols are supported (and what they are called) " + "may differ per provider.", + examples=["otlp_grpc", "otlp_http", "tempo_http"], + ) + + type: TransportProtocolType = pydantic.Field( + ..., + description="The transport protocol used by this receiver.", + examples=["http", "grpc"], + ) + +else: + + class ProtocolType(pydantic.BaseModel): + """Protocol Type.""" + + model_config = pydantic.ConfigDict( + # Allow serializing enum values. + use_enum_values=True + ) + """Pydantic config.""" + + name: str = pydantic.Field( + ..., + description="Receiver protocol name. What protocols are supported (and what they are called) " + "may differ per provider.", + examples=["otlp_grpc", "otlp_http", "tempo_http"], + ) + + type: TransportProtocolType = pydantic.Field( + ..., + description="The transport protocol used by this receiver.", + examples=["http", "grpc"], + ) + + +class Receiver(pydantic.BaseModel): + """Specification of an active receiver.""" + + protocol: ProtocolType = pydantic.Field(..., description="Receiver protocol name and type.") + url: str = pydantic.Field( + ..., + description="""URL at which the receiver is reachable. If there's an ingress, it would be the external URL. + Otherwise, it would be the service's fqdn or internal IP. + If the protocol type is grpc, the url will not contain a scheme.""", + examples=[ + "http://traefik_address:2331", + "https://traefik_address:2331", + "http://tempo_public_ip:2331", + "https://tempo_public_ip:2331", + "tempo_public_ip:2331", + ], + ) + + +class CosAgentRequirerUnitData(DatabagModel): # noqa: D101 + """Application databag model for the COS-agent requirer.""" + + receivers: List[Receiver] = pydantic.Field( + ..., + description="List of all receivers enabled on the tracing provider.", + ) + + class COSAgentProvider(Object): """Integration endpoint wrapper for the provider side of the cos_agent interface.""" @@ -318,6 +613,7 @@ def __init__( log_slots: Optional[List[str]] = None, dashboard_dirs: Optional[List[str]] = None, refresh_events: Optional[List] = None, + tracing_protocols: Optional[List[str]] = None, *, scrape_configs: Optional[Union[List[dict], Callable]] = None, ): @@ -336,6 +632,7 @@ def __init__( in the form ["snap-name:slot", ...]. dashboard_dirs: Directory where the dashboards are stored. refresh_events: List of events on which to refresh relation data. + tracing_protocols: List of protocols that the charm will be using for sending traces. scrape_configs: List of standard scrape_configs dicts or a callable that returns the list in case the configs need to be generated dynamically. The contents of this list will be merged with the contents of `metrics_endpoints`. @@ -353,6 +650,8 @@ def __init__( self._log_slots = log_slots or [] self._dashboard_dirs = dashboard_dirs self._refresh_events = refresh_events or [self._charm.on.config_changed] + self._tracing_protocols = tracing_protocols + self._is_single_endpoint = charm.meta.relations[relation_name].limit == 1 events = self._charm.on[relation_name] self.framework.observe(events.relation_joined, self._on_refresh) @@ -377,6 +676,7 @@ def _on_refresh(self, event): dashboards=self._dashboards, metrics_scrape_jobs=self._scrape_jobs, log_slots=self._log_slots, + tracing_protocols=self._tracing_protocols, ) relation.data[self._charm.unit][data.KEY] = data.json() except ( @@ -441,6 +741,103 @@ def _dashboards(self) -> List[GrafanaDashboard]: dashboards.append(dashboard) return dashboards + @property + def relations(self) -> List[Relation]: + """The tracing relations associated with this endpoint.""" + return self._charm.model.relations[self._relation_name] + + @property + def _relation(self) -> Optional[Relation]: + """If this wraps a single endpoint, the relation bound to it, if any.""" + if not self._is_single_endpoint: + objname = type(self).__name__ + raise AmbiguousRelationUsageError( + f"This {objname} wraps a {self._relation_name} endpoint that has " + "limit != 1. We can't determine what relation, of the possibly many, you are " + f"referring to. Please pass a relation instance while calling {objname}, " + "or set limit=1 in the charm metadata." + ) + relations = self.relations + return relations[0] if relations else None + + def is_ready(self, relation: Optional[Relation] = None): + """Is this endpoint ready?""" + relation = relation or self._relation + if not relation: + logger.debug(f"no relation on {self._relation_name !r}: tracing not ready") + return False + if relation.data is None: + logger.error(f"relation data is None for {relation}") + return False + if not relation.app: + logger.error(f"{relation} event received but there is no relation.app") + return False + try: + unit = next(iter(relation.units), None) + if not unit: + return False + databag = dict(relation.data[unit]) + CosAgentRequirerUnitData.load(databag) + + except (json.JSONDecodeError, pydantic.ValidationError, DataValidationError): + logger.info(f"failed validating relation data for {relation}") + return False + return True + + def get_all_endpoints( + self, relation: Optional[Relation] = None + ) -> Optional[CosAgentRequirerUnitData]: + """Unmarshalled relation data.""" + relation = relation or self._relation + if not relation or not self.is_ready(relation): + return None + unit = next(iter(relation.units), None) + if not unit: + return None + return CosAgentRequirerUnitData.load(relation.data[unit]) # type: ignore + + def _get_tracing_endpoint( + self, relation: Optional[Relation], protocol: ReceiverProtocol + ) -> Optional[str]: + unit_data = self.get_all_endpoints(relation) + if not unit_data: + return None + receivers: List[Receiver] = [i for i in unit_data.receivers if i.protocol.name == protocol] + if not receivers: + logger.error(f"no receiver found with protocol={protocol!r}") + return None + if len(receivers) > 1: + logger.error( + f"too many receivers with protocol={protocol!r}; using first one. Found: {receivers}" + ) + return None + + receiver = receivers[0] + return receiver.url + + def get_tracing_endpoint( + self, protocol: ReceiverProtocol, relation: Optional[Relation] = None + ) -> Optional[str]: + """Receiver endpoint for the given protocol.""" + endpoint = self._get_tracing_endpoint(relation or self._relation, protocol=protocol) + if not endpoint: + requested_protocols = set() + relations = [relation] if relation else self.relations + for relation in relations: + try: + databag = CosAgentProviderUnitData.load(relation.data[self._charm.unit]) + except DataValidationError: + continue + + if databag.tracing_protocols: + requested_protocols.update(databag.tracing_protocols) + + if protocol not in requested_protocols: + raise ProtocolNotRequestedError(protocol, relation) + + return None + return endpoint + class COSAgentDataChanged(EventBase): """Event emitted by `COSAgentRequirer` when relation data changes.""" @@ -554,6 +951,12 @@ def _on_relation_data_changed(self, event: RelationChangedEvent): if not (provider_data := self._validated_provider_data(raw)): return + # write enabled receivers to cos-agent relation + try: + self.update_tracing_receivers() + except ModelError: + raise + # Copy data from the cos_agent relation to the peer relation, so the leader could # follow up. # Save the originating unit name, so it could be used for topology later on by the leader. @@ -574,6 +977,37 @@ def _on_relation_data_changed(self, event: RelationChangedEvent): # need to emit `on.data_changed`), so we're emitting `on.data_changed` either way. self.on.data_changed.emit() # pyright: ignore + def update_tracing_receivers(self): + """Updates the list of exposed tracing receivers in all relations.""" + try: + for relation in self._charm.model.relations[self._relation_name]: + CosAgentRequirerUnitData( + receivers=[ + Receiver( + url=f"{self._get_tracing_receiver_url(protocol)}", + protocol=ProtocolType( + name=protocol, + type=receiver_protocol_to_transport_protocol[protocol], + ), + ) + for protocol in self.requested_tracing_protocols() + ], + ).dump(relation.data[self._charm.unit]) + + except ModelError as e: + # args are bytes + msg = e.args[0] + if isinstance(msg, bytes): + if msg.startswith( + b"ERROR cannot read relation application settings: permission denied" + ): + logger.error( + f"encountered error {e} while attempting to update_relation_data." + f"The relation must be gone." + ) + return + raise + def _validated_provider_data(self, raw) -> Optional[CosAgentProviderUnitData]: try: return CosAgentProviderUnitData(**json.loads(raw)) @@ -586,6 +1020,55 @@ def trigger_refresh(self, _): # FIXME: Figure out what we should do here self.on.data_changed.emit() # pyright: ignore + def _get_requested_protocols(self, relation: Relation): + # Coherence check + units = relation.units + if len(units) > 1: + # should never happen + raise ValueError( + f"unexpected error: subordinate relation {relation} " + f"should have exactly one unit" + ) + + unit = next(iter(units), None) + + if not unit: + return None + + if not (raw := relation.data[unit].get(CosAgentProviderUnitData.KEY)): + return None + + if not (provider_data := self._validated_provider_data(raw)): + return None + + return provider_data.tracing_protocols + + def requested_tracing_protocols(self): + """All receiver protocols that have been requested by our related apps.""" + requested_protocols = set() + for relation in self._charm.model.relations[self._relation_name]: + try: + protocols = self._get_requested_protocols(relation) + except NotReadyError: + continue + if protocols: + requested_protocols.update(protocols) + return requested_protocols + + def _get_tracing_receiver_url(self, protocol: str): + scheme = "http" + try: + if self._charm.cert.enabled: # type: ignore + scheme = "https" + # not only Grafana Agent can implement cos_agent. If the charm doesn't have the `cert` attribute + # using our cert_handler, it won't have the `enabled` parameter. In this case, we pass and assume http. + except AttributeError: + pass + # the assumption is that a subordinate charm will always be accessible to its principal charm under its fqdn + if receiver_protocol_to_transport_protocol[protocol] == TransportProtocolType.grpc: + return f"{socket.getfqdn()}:{_tracing_receivers_ports[protocol]}" + return f"{scheme}://{socket.getfqdn()}:{_tracing_receivers_ports[protocol]}" + @property def _remote_data(self) -> List[Tuple[CosAgentProviderUnitData, JujuTopology]]: """Return a list of remote data from each of the related units. @@ -721,8 +1204,18 @@ def metrics_jobs(self) -> List[Dict]: @property def snap_log_endpoints(self) -> List[SnapEndpoint]: """Fetch logging endpoints exposed by related snaps.""" + endpoints = [] + endpoints_with_topology = self.snap_log_endpoints_with_topology + for endpoint, _ in endpoints_with_topology: + endpoints.append(endpoint) + + return endpoints + + @property + def snap_log_endpoints_with_topology(self) -> List[Tuple[SnapEndpoint, JujuTopology]]: + """Fetch logging endpoints and charm topology for each related snap.""" plugs = [] - for data, _ in self._remote_data: + for data, topology in self._remote_data: targets = data.log_slots if targets: for target in targets: @@ -733,15 +1226,16 @@ def snap_log_endpoints(self) -> List[SnapEndpoint]: "endpoints; this should not happen." ) else: - plugs.append(target) + plugs.append((target, topology)) endpoints = [] - for plug in plugs: + for plug, topology in plugs: if ":" not in plug: logger.error(f"invalid plug definition received: {plug}. Ignoring...") else: endpoint = SnapEndpoint(*plug.split(":")) - endpoints.append(endpoint) + endpoints.append((endpoint, topology)) + return endpoints @property @@ -804,3 +1298,67 @@ def dashboards(self) -> List[Dict[str, str]]: ) return dashboards + + +def charm_tracing_config( + endpoint_requirer: COSAgentProvider, cert_path: Optional[Union[Path, str]] +) -> Tuple[Optional[str], Optional[str]]: + """Utility function to determine the charm_tracing config you will likely want. + + If no endpoint is provided: + disable charm tracing. + If https endpoint is provided but cert_path is not found on disk: + disable charm tracing. + If https endpoint is provided and cert_path is None: + ERROR + Else: + proceed with charm tracing (with or without tls, as appropriate) + + Usage: + If you are using charm_tracing >= v1.9: + >>> from lib.charms.tempo_k8s.v1.charm_tracing import trace_charm + >>> from lib.charms.tempo_k8s.v0.cos_agent import charm_tracing_config + >>> @trace_charm(tracing_endpoint="my_endpoint", cert_path="cert_path") + >>> class MyCharm(...): + >>> _cert_path = "/path/to/cert/on/charm/container.crt" + >>> def __init__(self, ...): + >>> self.cos_agent = COSAgentProvider(...) + >>> self.my_endpoint, self.cert_path = charm_tracing_config( + ... self.cos_agent, self._cert_path) + + If you are using charm_tracing < v1.9: + >>> from lib.charms.tempo_k8s.v1.charm_tracing import trace_charm + >>> from lib.charms.tempo_k8s.v2.tracing import charm_tracing_config + >>> @trace_charm(tracing_endpoint="my_endpoint", cert_path="cert_path") + >>> class MyCharm(...): + >>> _cert_path = "/path/to/cert/on/charm/container.crt" + >>> def __init__(self, ...): + >>> self.cos_agent = COSAgentProvider(...) + >>> self.my_endpoint, self.cert_path = charm_tracing_config( + ... self.cos_agent, self._cert_path) + >>> @property + >>> def my_endpoint(self): + >>> return self._my_endpoint + >>> @property + >>> def cert_path(self): + >>> return self._cert_path + + """ + if not endpoint_requirer.is_ready(): + return None, None + + endpoint = endpoint_requirer.get_tracing_endpoint("otlp_http") + if not endpoint: + return None, None + + is_https = endpoint.startswith("https://") + + if is_https: + if cert_path is None: + raise TracingError("Cannot send traces to an https endpoint without a certificate.") + if not Path(cert_path).exists(): + # if endpoint is https BUT we don't have a server_cert yet: + # disable charm tracing until we do to prevent tls errors + return None, None + return endpoint, str(cert_path) + return endpoint, None diff --git a/lib/charms/tempo_k8s/v1/charm_tracing.py b/lib/charms/tempo_k8s/v1/charm_tracing.py index 4306b4d8b..fa926539b 100644 --- a/lib/charms/tempo_k8s/v1/charm_tracing.py +++ b/lib/charms/tempo_k8s/v1/charm_tracing.py @@ -172,14 +172,65 @@ def my_tracing_endpoint(self) -> Optional[str]: provide an *absolute* path to the certificate file instead. """ + +def _remove_stale_otel_sdk_packages(): + """Hack to remove stale opentelemetry sdk packages from the charm's python venv. + + See https://github.com/canonical/grafana-agent-operator/issues/146 and + https://bugs.launchpad.net/juju/+bug/2058335 for more context. This patch can be removed after + this juju issue is resolved and sufficient time has passed to expect most users of this library + have migrated to the patched version of juju. When this patch is removed, un-ignore rule E402 for this file in the pyproject.toml (see setting + [tool.ruff.lint.per-file-ignores] in pyproject.toml). + + This only has an effect if executed on an upgrade-charm event. + """ + # all imports are local to keep this function standalone, side-effect-free, and easy to revert later + import os + + if os.getenv("JUJU_DISPATCH_PATH") != "hooks/upgrade-charm": + return + + import logging + import shutil + from collections import defaultdict + + from importlib_metadata import distributions + + otel_logger = logging.getLogger("charm_tracing_otel_patcher") + otel_logger.debug("Applying _remove_stale_otel_sdk_packages patch on charm upgrade") + # group by name all distributions starting with "opentelemetry_" + otel_distributions = defaultdict(list) + for distribution in distributions(): + name = distribution._normalized_name # type: ignore + if name.startswith("opentelemetry_"): + otel_distributions[name].append(distribution) + + otel_logger.debug(f"Found {len(otel_distributions)} opentelemetry distributions") + + # If we have multiple distributions with the same name, remove any that have 0 associated files + for name, distributions_ in otel_distributions.items(): + if len(distributions_) <= 1: + continue + + otel_logger.debug(f"Package {name} has multiple ({len(distributions_)}) distributions.") + for distribution in distributions_: + if not distribution.files: # Not None or empty list + path = distribution._path # type: ignore + otel_logger.info(f"Removing empty distribution of {name} at {path}.") + shutil.rmtree(path) + + otel_logger.debug("Successfully applied _remove_stale_otel_sdk_packages patch. ") + + +_remove_stale_otel_sdk_packages() + + import functools import inspect import logging import os -import shutil from contextlib import contextmanager from contextvars import Context, ContextVar, copy_context -from importlib.metadata import distributions from pathlib import Path from typing import ( Any, @@ -199,14 +250,15 @@ def my_tracing_endpoint(self) -> Optional[str]: from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import Span, TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.trace import INVALID_SPAN, Tracer -from opentelemetry.trace import get_current_span as otlp_get_current_span from opentelemetry.trace import ( + INVALID_SPAN, + Tracer, get_tracer, get_tracer_provider, set_span_in_context, set_tracer_provider, ) +from opentelemetry.trace import get_current_span as otlp_get_current_span from ops.charm import CharmBase from ops.framework import Framework @@ -219,7 +271,7 @@ def my_tracing_endpoint(self) -> Optional[str]: # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 13 +LIBPATCH = 14 PYDEPS = ["opentelemetry-exporter-otlp-proto-http==1.21.0"] @@ -361,30 +413,6 @@ def _get_server_cert( return server_cert -def _remove_stale_otel_sdk_packages(): - """Hack to remove stale opentelemetry sdk packages from the charm's python venv. - - See https://github.com/canonical/grafana-agent-operator/issues/146 and - https://bugs.launchpad.net/juju/+bug/2058335 for more context. This patch can be removed after - this juju issue is resolved and sufficient time has passed to expect most users of this library - have migrated to the patched version of juju. - - This only does something if executed on an upgrade-charm event. - """ - if os.getenv("JUJU_DISPATCH_PATH") == "hooks/upgrade-charm": - logger.debug("Executing _remove_stale_otel_sdk_packages patch on charm upgrade") - # Find any opentelemetry_sdk distributions - otel_sdk_distributions = list(distributions(name="opentelemetry_sdk")) - # If there is more than 1, inspect each and if it has 0 entrypoints, infer that it is stale - if len(otel_sdk_distributions) > 1: - for distribution in otel_sdk_distributions: - if len(distribution.entry_points) == 0: - # Distribution appears to be empty. Remove it - path = distribution._path # type: ignore - logger.debug(f"Removing empty opentelemetry_sdk distribution at: {path}") - shutil.rmtree(path) - - def _setup_root_span_initializer( charm_type: _CharmType, tracing_endpoint_attr: str, @@ -420,7 +448,6 @@ def wrap_init(self: CharmBase, framework: Framework, *args, **kwargs): # apply hacky patch to remove stale opentelemetry sdk packages on upgrade-charm. # it could be trouble if someone ever decides to implement their own tracer parallel to # ours and before the charm has inited. We assume they won't. - _remove_stale_otel_sdk_packages() resource = Resource.create( attributes={ "service.name": _service_name, diff --git a/metadata.yaml b/metadata.yaml index aa15ee1f1..515ccdc72 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -71,3 +71,8 @@ requires: interface: tracing limit: 1 optional: true + + ha: + interface: hacluster + limit: 1 + optional: true diff --git a/src/charm.py b/src/charm.py index c90a3e49d..00585197d 100755 --- a/src/charm.py +++ b/src/charm.py @@ -24,17 +24,18 @@ from charms.tempo_k8s.v1.charm_tracing import trace_charm from charms.tempo_k8s.v2.tracing import TracingEndpointRequirer from jinja2 import Template -from ops import JujuVersion -from ops.charm import CharmBase, StartEvent -from ops.main import main -from ops.model import ( +from ops import ( ActiveStatus, BlockedStatus, + CharmBase, + JujuVersion, MaintenanceStatus, ModelError, Relation, + StartEvent, WaitingStatus, ) +from ops.main import main from constants import ( APP_SCOPE, @@ -65,6 +66,7 @@ ) from relations.backend_database import BackendDatabaseRequires from relations.db import DbProvides +from relations.hacluster import HaCluster from relations.peers import Peers from relations.pgbouncer_provider import PgBouncerProvider from upgrade import PgbouncerUpgrade, get_pgbouncer_dependencies_model @@ -120,6 +122,7 @@ def __init__(self, *args): self.legacy_db_relation = DbProvides(self, admin=False) self.legacy_db_admin_relation = DbProvides(self, admin=True) self.tls = PostgreSQLTLS(self, PEER_RELATION_NAME) + self.hacluster = HaCluster(self) self.service_ids = list(range(self.instances_count)) self.pgb_services = [ @@ -507,8 +510,24 @@ def update_status(self): self.unit.status = BlockedStatus("backend database relation not ready") return + if self.hacluster.relation and not self._is_exposed: + self.unit.status = BlockedStatus("ha integration used without data-intgrator") + return + + vip = self.config.get("vip") + if self.hacluster.relation and not vip: + self.unit.status = BlockedStatus("ha integration used without vip configuration") + return + + if vip and not self._is_exposed: + self.unit.status = BlockedStatus("vip configuration without data-intgrator") + return + if self.check_pgb_running(): - self.unit.status = ActiveStatus() + if self.unit.is_leader() and vip: + self.unit.status = ActiveStatus(f"VIP: {vip}") + else: + self.unit.status = ActiveStatus() def _on_config_changed(self, event) -> None: """Config changed handler. @@ -521,19 +540,28 @@ def _on_config_changed(self, event) -> None: event.defer() return + old_vip = self.peers.app_databag.get("current_vip", "") + vip = self.config.get("vip", "") + vip_changed = old_vip != vip + if vip_changed and self._is_exposed: + self.hacluster.set_vip(self.config.get("vip")) + old_port = self.peers.app_databag.get("current_port") port_changed = old_port != str(self.config["listen_port"]) if port_changed and self._is_exposed: - if self.unit.is_leader(): - self.peers.app_databag["current_port"] = str(self.config["listen_port"]) # Open port try: - if old_port: - self.unit.close_port("tcp", old_port) - self.unit.open_port("tcp", self.config["listen_port"]) + self.unit.set_ports(self.config["listen_port"]) except ModelError: logger.exception("failed to open port") + if self.unit.is_leader(): + self.peers.app_databag["current_port"] = str(self.config["listen_port"]) + if vip: + self.peers.app_databag["current_vip"] = str(vip) + else: + self.peers.app_databag.pop("current_vip", None) + # TODO hitting upgrade errors here due to secrets labels failing to set on non-leaders. # deferring until the leader manages to set the label try: @@ -546,6 +574,9 @@ def _on_config_changed(self, event) -> None: if self.backend.postgres: self.render_prometheus_service() + if port_changed or vip_changed: + self.update_client_connection_info() + def check_pgb_running(self): """Checks that pgbouncer service is running, and updates status accordingly.""" prom_service = f"{PGB}-{self.app.name}-prometheus" @@ -910,14 +941,13 @@ def unit_ip(self) -> str: # Relation Utilities # ===================== - def update_client_connection_info(self, port: Optional[str] = None): + def update_client_connection_info(self): """Update ports in backend relations to match updated pgbouncer port.""" # Skip updates if backend.postgres doesn't exist yet. if not self.backend.postgres or not self.unit.is_leader(): return - if port is None: - port = self.config["listen_port"] + port = self.config["listen_port"] for relation in self.model.relations.get("db", []): self.legacy_db_relation.update_connection_info(relation, port) diff --git a/src/constants.py b/src/constants.py index 21289974e..aeb82b6b6 100644 --- a/src/constants.py +++ b/src/constants.py @@ -36,6 +36,7 @@ BACKEND_RELATION_NAME = "backend-database" PEER_RELATION_NAME = "pgb-peers" CLIENT_RELATION_NAME = "database" +HACLUSTER_RELATION_NAME = "ha" TLS_KEY_FILE = "key.pem" TLS_CA_FILE = "ca.pem" diff --git a/src/relations/hacluster.py b/src/relations/hacluster.py new file mode 100644 index 000000000..914cb0461 --- /dev/null +++ b/src/relations/hacluster.py @@ -0,0 +1,81 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +"""hacluster relation hooks and helpers.""" + +import json +import logging +from hashlib import shake_128 +from ipaddress import IPv4Address, ip_address +from typing import Optional + +from ops import CharmBase, Object, Relation, RelationChangedEvent, Unit + +from constants import HACLUSTER_RELATION_NAME + +logger = logging.getLogger(__name__) + + +class HaCluster(Object): + """Defines hacluster functunality.""" + + def __init__(self, charm: CharmBase): + super().__init__(charm, HACLUSTER_RELATION_NAME) + + self.charm = charm + + self.framework.observe( + charm.on[HACLUSTER_RELATION_NAME].relation_changed, self._on_changed + ) + + @property + def relation(self) -> Relation: + """Returns the relations in this model, or None if hacluster is not initialised.""" + return self.charm.model.get_relation(HACLUSTER_RELATION_NAME) + + def _is_clustered(self) -> bool: + for key, value in self.relation.data.items(): + if isinstance(key, Unit) and key != self.charm.unit: + if value.get("clustered") in ("yes", "true"): + return True + break + return False + + def _on_changed(self, event: RelationChangedEvent) -> None: + self.set_vip(self.charm.config.get("vip")) + + def set_vip(self, vip: Optional[str]) -> None: + """Adds the requested virtual IP to the integration.""" + if not self.relation: + return + + if not self._is_clustered(): + logger.debug("early exit set_vip: ha relation not yet clustered") + return + + if vip: + # TODO Add nic support + ipaddr = ip_address(vip) + vip_key = f"res_{self.charm.app.name}_{shake_128(vip.encode()).hexdigest(7)}_vip" + vip_params = " params" + if isinstance(ipaddr, IPv4Address): + vip_resources = "ocf:heartbeat:IPaddr2" + vip_params += f' ip="{vip}"' + else: + vip_resources = "ocf:heartbeat:IPv6addr" + vip_params += f' ipv6addr="{vip}"' + + # Monitor the VIP + vip_params += ' meta migration-threshold="INFINITY" failure-timeout="5s"' + vip_params += ' op monitor timeout="20s" interval="10s" depth="0"' + json_resources = json.dumps({vip_key: vip_resources}) + json_resource_params = json.dumps({vip_key: vip_params}) + + else: + json_resources = "{}" + json_resource_params = "{}" + + self.relation.data[self.charm.unit].update({ + "json_resources": json_resources, + "json_resource_params": json_resource_params, + }) + self.charm.update_status() diff --git a/src/relations/pgbouncer_provider.py b/src/relations/pgbouncer_provider.py index 77c79e9ba..a22b93b98 100644 --- a/src/relations/pgbouncer_provider.py +++ b/src/relations/pgbouncer_provider.py @@ -239,7 +239,11 @@ def update_connection_info(self, relation): f"Updating {self.relation_name} relation connection information" ) if exposed: - rw_endpoint = f"{self.charm.leader_ip}:{self.charm.config['listen_port']}" + if vip := self.charm.config.get("vip"): + ip = vip + else: + ip = self.charm.leader_ip + rw_endpoint = f"{ip}:{self.charm.config['listen_port']}" else: rw_endpoint = f"localhost:{self.charm.config['listen_port']}" self.database_provides.set_endpoints( @@ -265,7 +269,7 @@ def set_ready(self) -> None: def update_read_only_endpoints(self, event: DatabaseRequestedEvent = None) -> None: """Set the read-only endpoint only if there are replicas.""" - if not self.charm.unit.is_leader(): + if not self.charm.unit.is_leader() or self.charm.config.get("vip"): return # Get the current relation or all the relations if this is triggered by another type of diff --git a/tests/integration/relations/pgbouncer_provider/test_data_integrator_ha.py b/tests/integration/relations/pgbouncer_provider/test_data_integrator_ha.py new file mode 100644 index 000000000..460dd8741 --- /dev/null +++ b/tests/integration/relations/pgbouncer_provider/test_data_integrator_ha.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +import asyncio +import logging + +import pytest +from pytest_operator.plugin import OpsTest + +from constants import BACKEND_RELATION_NAME + +from ... import architecture +from ...helpers.ha_helpers import get_unit_ip +from ...helpers.helpers import ( + PG, + PGB, +) +from ...juju_ import juju_major_version +from .helpers import check_exposed_connection, fetch_action_get_credentials + +logger = logging.getLogger(__name__) + +DATA_INTEGRATOR_APP_NAME = "data-integrator" +HACLUSTER_NAME = "hacluster" + +if juju_major_version < 3: + tls_certificates_app_name = "tls-certificates-operator" + if architecture.architecture == "arm64": + tls_channel = "legacy/edge" + else: + tls_channel = "legacy/stable" + tls_config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} +else: + tls_certificates_app_name = "self-signed-certificates" + if architecture.architecture == "arm64": + tls_channel = "latest/edge" + else: + tls_channel = "latest/stable" + tls_config = {"ca-common-name": "Test CA"} + + +@pytest.mark.group(1) +@pytest.mark.abort_on_fail +async def test_deploy_and_relate(ops_test: OpsTest, pgb_charm_jammy): + """Test basic functionality of database relation interface.""" + # Deploy both charms (multiple units for each application to test that later they correctly + # set data in the relation application databag using only the leader unit). + config = {"database-name": "test-database"} + async with ops_test.fast_forward(): + await asyncio.gather( + ops_test.model.deploy( + pgb_charm_jammy, + application_name=PGB, + num_units=None, + ), + ops_test.model.deploy( + PG, + application_name=PG, + num_units=2, + channel="14/edge", + config={"profile": "testing"}, + ), + ops_test.model.deploy( + DATA_INTEGRATOR_APP_NAME, + channel="edge", + num_units=3, + config=config, + ), + ops_test.model.deploy( + tls_certificates_app_name, config=tls_config, channel=tls_channel + ), + ops_test.model.deploy( + HACLUSTER_NAME, + channel="2.4/stable", + num_units=0, + ), + ) + await ops_test.model.add_relation(f"{PGB}:{BACKEND_RELATION_NAME}", f"{PG}:database") + await ops_test.model.add_relation(f"{PGB}:ha", f"{HACLUSTER_NAME}:ha") + await ops_test.model.add_relation( + f"{DATA_INTEGRATOR_APP_NAME}:juju-info", f"{HACLUSTER_NAME}:juju-info" + ) + + await ops_test.model.wait_for_idle(apps=[PG], status="active", timeout=1200) + ip_addresses = [ + await get_unit_ip(ops_test, unit_name) + for unit_name in ops_test.model.units.keys() + if unit_name.startswith(DATA_INTEGRATOR_APP_NAME) or unit_name.startswith(PG) + ] + + # Try to generate a vip + base, last_octet = ip_addresses[0].rsplit(".", 1) + last_octet = int(last_octet) + global vip + vip = None + for _ in range(len(ip_addresses)): + last_octet += 1 + if last_octet > 254: + last_octet = 2 + addr = ".".join([base, str(last_octet)]) + if addr not in ip_addresses: + vip = addr + break + logger.info(f"Setting VIP to {vip}") + + await ops_test.model.applications[PGB].set_config({"vip": vip}) + await ops_test.model.add_relation(PGB, DATA_INTEGRATOR_APP_NAME) + await ops_test.model.wait_for_idle(status="active", timeout=600) + + credentials = await fetch_action_get_credentials( + ops_test.model.applications[DATA_INTEGRATOR_APP_NAME].units[0] + ) + check_exposed_connection(credentials, False) + host, _ = credentials["postgresql"]["endpoints"].split(":") + logger.info(f"Data integrator host is {host}") + assert host == vip + + +@pytest.mark.group(1) +async def test_add_tls(ops_test: OpsTest, pgb_charm_jammy): + await ops_test.model.add_relation(PGB, tls_certificates_app_name) + await ops_test.model.wait_for_idle(status="active") + + credentials = await fetch_action_get_credentials( + ops_test.model.applications[DATA_INTEGRATOR_APP_NAME].units[0] + ) + check_exposed_connection(credentials, True) + + +@pytest.mark.group(1) +async def test_remove_tls(ops_test: OpsTest, pgb_charm_jammy): + await ops_test.model.applications[PGB].remove_relation( + f"{PGB}:certificates", f"{tls_certificates_app_name}:certificates" + ) + await ops_test.model.wait_for_idle(status="active") + + credentials = await fetch_action_get_credentials( + ops_test.model.applications[DATA_INTEGRATOR_APP_NAME].units[0] + ) + check_exposed_connection(credentials, False) + + +@pytest.mark.group(1) +async def test_remove_vip(ops_test: OpsTest): + async with ops_test.fast_forward(): + await ops_test.model.applications[PGB].set_config({"vip": ""}) + await ops_test.model.wait_for_idle(apps=[PGB], status="blocked", timeout=300) + assert ( + ops_test.model.applications[PGB].units[0].workload_status_message + == "ha integration used without vip configuration" + ) + + await ops_test.model.applications[PGB].remove_relation(f"{PGB}:ha", f"{HACLUSTER_NAME}:ha") + await ops_test.model.wait_for_idle(apps=[PGB], status="active", timeout=600) + + credentials = await fetch_action_get_credentials( + ops_test.model.applications[DATA_INTEGRATOR_APP_NAME].units[0] + ) + host, _ = credentials["postgresql"]["endpoints"].split(":") + logger.info(f"Data integrator host is {host}") + assert host != vip diff --git a/tests/unit/relations/test_hacluster.py b/tests/unit/relations/test_hacluster.py new file mode 100644 index 000000000..ff90c2025 --- /dev/null +++ b/tests/unit/relations/test_hacluster.py @@ -0,0 +1,100 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +from unittest import TestCase +from unittest.mock import Mock, PropertyMock, patch + +from ops.testing import Harness + +from charm import PgBouncerCharm +from constants import HACLUSTER_RELATION_NAME + +from ..helpers import patch_network_get + + +@patch_network_get(private_address="1.1.1.1") +class TestHaCluster(TestCase): + def setUp(self): + self.harness = Harness(PgBouncerCharm) + self.addCleanup(self.harness.cleanup) + self.harness.begin() + + self.charm = self.harness.charm + self.app = self.charm.app.name + self.unit = self.charm.unit.name + self.harness.add_relation("upgrade", self.charm.app.name) + + self.rel_id = self.harness.add_relation(HACLUSTER_RELATION_NAME, self.charm.app.name) + + def test_is_clustered(self): + # No remote + assert not self.charm.hacluster._is_clustered() + + # Not clustered + with self.harness.hooks_disabled(): + self.harness.add_relation_unit(self.rel_id, "hacluster/0") + self.harness.update_relation_data(self.rel_id, "hacluster/0", {}) + + assert not self.charm.hacluster._is_clustered() + + # Valid clustered + with self.harness.hooks_disabled(): + self.harness.update_relation_data(self.rel_id, "hacluster/0", {"clustered": "yes"}) + + assert self.charm.hacluster._is_clustered() + + @patch("charm.HaCluster.set_vip", return_value=True) + def test_on_changed(self, _set_vip): + with self.harness.hooks_disabled(): + self.harness.update_config({"vip": "test_vip"}) + + self.charm.hacluster._on_changed(Mock()) + + _set_vip.assert_called_once_with("test_vip") + + @patch("charm.HaCluster._is_clustered", return_value=False) + @patch("charm.HaCluster.relation", new_callable=PropertyMock, return_value=False) + def test_set_vip_no_relation(self, _relation, _is_clustered): + # Not rel + self.charm.hacluster.set_vip("1.2.3.4") + + assert not _is_clustered.called + + @patch("charm.HaCluster._is_clustered", return_value=False) + def test_set_vip(self, _is_clustered): + # Not clustered + self.charm.hacluster.set_vip("1.2.3.4") + assert self.harness.get_relation_data(self.rel_id, self.charm.unit) == {} + + # ipv4 address + _is_clustered.return_value = True + + self.charm.hacluster.set_vip("1.2.3.4") + + assert self.harness.get_relation_data(self.rel_id, self.charm.unit) == { + "json_resource_params": '{"res_pgbouncer_d716ce1885885a_vip": " params ' + 'ip=\\"1.2.3.4\\" meta ' + 'migration-threshold=\\"INFINITY\\" ' + 'failure-timeout=\\"5s\\" op monitor ' + 'timeout=\\"20s\\" interval=\\"10s\\" depth=\\"0\\""}', + "json_resources": '{"res_pgbouncer_d716ce1885885a_vip": ' '"ocf:heartbeat:IPaddr2"}', + } + + # ipv6 address + self.charm.hacluster.set_vip("::1") + + assert self.harness.get_relation_data(self.rel_id, self.charm.unit) == { + "json_resource_params": '{"res_pgbouncer_61b6532057c944_vip": " params ' + 'ipv6addr=\\"::1\\" meta ' + 'migration-threshold=\\"INFINITY\\" ' + 'failure-timeout=\\"5s\\" op monitor ' + 'timeout=\\"20s\\" interval=\\"10s\\" depth=\\"0\\""}', + "json_resources": '{"res_pgbouncer_61b6532057c944_vip": ' '"ocf:heartbeat:IPv6addr"}', + } + + # unset data + self.charm.hacluster.set_vip("") + assert self.harness.get_relation_data(self.rel_id, self.charm.unit) == { + "json_resource_params": "{}", + "json_resources": "{}", + } diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index f3eac7e16..022f8241f 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -26,6 +26,7 @@ from constants import ( AUTH_FILE_NAME, BACKEND_RELATION_NAME, + EXTENSIONS_BLOCKING_MESSAGE, PEER_RELATION_NAME, PGB_CONF_DIR, PGB_LOG_DIR, @@ -584,6 +585,99 @@ def test_collect_readonly_dbs(self, _get_relation_databases, _postgres): assert self.charm.peers.app_databag["readonly_dbs"] == '["includeddb"]' + @patch("charm.HaCluster.relation", new_callable=PropertyMock, return_value=True) + @patch("charm.PgBouncerCharm._is_exposed", new_callable=PropertyMock, return_value=False) + @patch( + "relations.backend_database.BackendDatabaseRequires.ready", + new_callable=PropertyMock, + return_value=False, + ) + @patch("charm.BackendDatabaseRequires.postgres", new_callable=PropertyMock, return_value=None) + @patch("charm.PgBouncerCharm.check_pgb_running", return_value=True) + def test_update_status(self, _check_pgb_running, _postgres, _ready, _is_exposed, _ha_relation): + # Doesn't clear extensions blocking + self.charm.unit.status = BlockedStatus(EXTENSIONS_BLOCKING_MESSAGE) + + self.charm.update_status() + + assert isinstance(self.charm.unit.status, BlockedStatus) + assert self.charm.unit.status.message == EXTENSIONS_BLOCKING_MESSAGE + + # Blocks if no backend + self.charm.unit.status = ActiveStatus() + + self.charm.update_status() + + assert isinstance(self.charm.unit.status, BlockedStatus) + assert ( + self.charm.unit.status.message == "waiting for backend database relation to initialise" + ) + + # Blocks if backend is not ready + self.charm.unit.status = ActiveStatus() + _postgres.return_value = True + + self.charm.update_status() + + assert isinstance(self.charm.unit.status, BlockedStatus) + assert self.charm.unit.status.message == "backend database relation not ready" + + # Blocks if using hacluster and not exposed + self.charm.unit.status = ActiveStatus() + _ready.return_value = True + + self.charm.update_status() + + assert isinstance(self.charm.unit.status, BlockedStatus) + assert self.charm.unit.status.message == "ha integration used without data-intgrator" + + # Blocks if using hacluster and not set vip + self.charm.unit.status = ActiveStatus() + _is_exposed.return_value = True + + self.charm.update_status() + + assert isinstance(self.charm.unit.status, BlockedStatus) + assert self.charm.unit.status.message == "ha integration used without vip configuration" + + # Blocks if vip is set and not exposed + self.charm.unit.status = ActiveStatus() + _is_exposed.return_value = False + _ha_relation.return_value = False + with self.harness.hooks_disabled(): + self.harness.update_config({"vip": "1.2.3.4"}) + + self.charm.update_status() + + assert isinstance(self.charm.unit.status, BlockedStatus) + assert self.charm.unit.status.message == "vip configuration without data-intgrator" + + # Unblocks if running check passes + self.charm.unit.status = BlockedStatus() + _is_exposed.return_value = True + _ha_relation.return_value = True + + self.charm.update_status() + + assert isinstance(self.charm.unit.status, ActiveStatus) + + # Keeps status if checks don't pass + self.charm.unit.status = BlockedStatus() + _check_pgb_running.return_value = False + + self.charm.update_status() + + assert isinstance(self.charm.unit.status, BlockedStatus) + + # Leader sets vip in unit status + _check_pgb_running.return_value = True + with self.harness.hooks_disabled(): + self.harness.set_leader() + + self.charm.update_status() + + assert self.charm.unit.status.message == "VIP: 1.2.3.4" + # # Secrets #