diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index 9071655a..bf7ceb7e 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -298,7 +298,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): from collections import namedtuple from datetime import datetime from enum import Enum -from typing import Dict, List, Optional, Set, Union +from typing import Callable, Dict, List, Optional, Set, Tuple, Union from ops import JujuVersion, Secret, SecretInfo, SecretNotFoundError from ops.charm import ( @@ -320,7 +320,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 20 +LIBPATCH = 21 PYDEPS = ["ops>=2.0.0"] @@ -377,11 +377,24 @@ class SecretsIllegalUpdateError(SecretError): """Secrets aren't yet available for Juju version used.""" -def get_encoded_field( +def get_encoded_dict( relation: Relation, member: Union[Unit, Application], field: str -) -> Union[str, List[str], Dict[str, str]]: +) -> Optional[Dict[str, str]]: """Retrieve and decode an encoded field from relation data.""" - return json.loads(relation.data[member].get(field, "{}")) + data = json.loads(relation.data[member].get(field, "{}")) + if isinstance(data, dict): + return data + logger.error("Unexpected datatype for {data} instead of dict.") + + +def get_encoded_list( + relation: Relation, member: Union[Unit, Application], field: str +) -> Optional[List[str]]: + """Retrieve and decode an encoded field from relation data.""" + data = json.loads(relation.data[member].get(field, "[]")) + if isinstance(data, list): + return data + logger.error("Unexpected datatype for {data} instead of list.") def set_encoded_field( @@ -406,16 +419,11 @@ def diff(event: RelationChangedEvent, bucket: Union[Unit, Application]) -> Diff: keys from the event relation databag. """ # Retrieve the old data from the data key in the application relation databag. - old_data = get_encoded_field(event.relation, bucket, "data") + old_data = get_encoded_dict(event.relation, bucket, "data") if not old_data: old_data = {} - if not isinstance(old_data, dict): - # We should never get here, added to re-assure pyright - logger.error("Previous databag diff is of a wrong type.") - old_data = {} - # Retrieve the new data from the event relation databag. new_data = ( {key: value for key, value in event.relation.data[event.app].items() if key != "data"} @@ -523,9 +531,14 @@ def get_content(self) -> Dict[str, str]: def set_content(self, content: Dict[str, str]) -> None: """Setting cached secret content.""" - if self.meta: + if not self.meta: + return + + if content: self.meta.set_content(content) self._secret_content = content + else: + self.meta.remove_all_revisions() def get_info(self) -> Optional[SecretInfo]: """Wrapper function to apply the corresponding call on the Secret object within CachedSecret if any.""" @@ -622,6 +635,16 @@ def _fetch_my_specific_relation_data( """Fetch data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" raise NotImplementedError + @abstractmethod + def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: + """Update data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + raise NotImplementedError + + @abstractmethod + def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: + """Delete data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + raise NotImplementedError + # Internal helper methods @staticmethod @@ -688,9 +711,9 @@ def _group_secret_fields(secret_fields: List[str]) -> Dict[SecretGroup, List[str secret_fieldnames_grouped.setdefault(SecretGroup.EXTRA, []).append(key) return secret_fieldnames_grouped - def _retrieve_group_secret_contents( + def _get_group_secret_contents( self, - relation_id: int, + relation: Relation, group: SecretGroup, secret_fields: Optional[Union[Set[str], List[str]]] = None, ) -> Dict[str, str]: @@ -698,12 +721,29 @@ def _retrieve_group_secret_contents( if not secret_fields: secret_fields = [] - if (secret := self._get_relation_secret(relation_id, group)) and ( + if (secret := self._get_relation_secret(relation.id, group)) and ( secret_data := secret.get_content() ): return {k: v for k, v in secret_data.items() if k in secret_fields} return {} + @staticmethod + def _secret_content_grouped( + content: Dict[str, str], secret_fields: Set[str], group_mapping: SecretGroup + ) -> Dict[str, str]: + if group_mapping == SecretGroup.EXTRA: + return { + k: v + for k, v in content.items() + if k in secret_fields and k not in SECRET_LABEL_MAP.keys() + } + + return { + k: v + for k, v in content.items() + if k in secret_fields and SECRET_LABEL_MAP.get(k) == group_mapping + } + @juju_secrets_only def _get_relation_secret_data( self, relation_id: int, group_mapping: SecretGroup, relation_name: Optional[str] = None @@ -713,6 +753,38 @@ def _get_relation_secret_data( if secret: return secret.get_content() + # Core operations on Relation Fields manipulations (regardless whether the field is in the databag or in a secret) + # Internal functions to be called directly from transparent public interface functions (+closely related helpers) + + def _process_secret_fields( + self, + relation: Relation, + req_secret_fields: Optional[List[str]], + impacted_rel_fields: List[str], + operation: Callable, + second_chance_as_normal_field: bool = True, + *args, + **kwargs, + ) -> Tuple[Dict[str, str], Set[str]]: + """Isolate target secret fields of manipulation, and execute requested operation by Secret Group.""" + result = {} + normal_fields = set(impacted_rel_fields) + if req_secret_fields and self.secrets_enabled: + normal_fields = normal_fields - set(req_secret_fields) + secret_fields = set(impacted_rel_fields) - set(normal_fields) + + secret_fieldnames_grouped = self._group_secret_fields(list(secret_fields)) + + for group in secret_fieldnames_grouped: + # operation() should return nothing when all goes well + if group_result := operation(relation, group, secret_fields, *args, **kwargs): + result.update(group_result) + elif second_chance_as_normal_field: + # If it wasn't found as a secret, let's give it a 2nd chance as "normal" field + # Needed when Juju3 Requires meets Juju2 Provider + normal_fields |= set(secret_fieldnames_grouped[group]) + return (result, normal_fields) + def _fetch_relation_data_without_secrets( self, app: Application, relation: Relation, fields: Optional[List[str]] ) -> Dict[str, str]: @@ -743,43 +815,50 @@ def _fetch_relation_data_with_secrets( Provides side itself). """ result = {} + normal_fields = [] - normal_fields = fields - if not normal_fields: - normal_fields = list(relation.data[app].keys()) + if not fields: + all_fields = list(relation.data[app].keys()) + normal_fields = [field for field in all_fields if not self._is_secret_field(field)] - if req_secret_fields and self.secrets_enabled: - if fields: - # Processing from what was requested - normal_fields = set(fields) - set(req_secret_fields) - secret_fields = set(fields) - set(normal_fields) - - secret_fieldnames_grouped = self._group_secret_fields(list(secret_fields)) - - for group in secret_fieldnames_grouped: - if contents := self._retrieve_group_secret_contents( - relation.id, group, secret_fields - ): - result.update(contents) - else: - # If it wasn't found as a secret, let's give it a 2nd chance as "normal" field - normal_fields |= set(secret_fieldnames_grouped[group]) - else: - # Processing from what is given, i.e. retrieving all - normal_fields = [ - f for f in relation.data[app].keys() if not self._is_secret_field(f) - ] - secret_fields = [f for f in relation.data[app].keys() if self._is_secret_field(f)] - for group in SecretGroup: - result.update( - self._retrieve_group_secret_contents(relation.id, group, req_secret_fields) - ) + # There must have been secrets there + if all_fields != normal_fields and req_secret_fields: + # So we assemble the full fields list (without 'secret-' fields) + fields = normal_fields + req_secret_fields + + if fields: + result, normal_fields = self._process_secret_fields( + relation, req_secret_fields, fields, self._get_group_secret_contents + ) # Processing "normal" fields. May include leftover from what we couldn't retrieve as a secret. - result.update({k: relation.data[app][k] for k in normal_fields if k in relation.data[app]}) + # (Typically when Juju3 Requires meets Juju2 Provides) + if normal_fields: + result.update( + self._fetch_relation_data_without_secrets(app, relation, list(normal_fields)) + ) return result - # Public methods + def _update_relation_data_without_secrets( + self, app: Application, relation: Relation, data: Dict[str, str] + ): + """Updating databag contents when no secrets are involved.""" + if any(self._is_secret_field(key) for key in data.keys()): + raise SecretsIllegalUpdateError("Can't update secret {key}.") + + if relation: + relation.data[app].update(data) + + @leader_only + def _delete_relation_data_without_secrets( + self, app: Application, relation: Relation, fields: List[str] + ) -> None: + """Remove databag fields 'fields' from Relation.""" + for field in fields: + relation.data[app].pop(field) + + # Public interface methods + # Handling Relation Fields seamlessly, regardless if in databag or a Juju Secret def get_relation(self, relation_name, relation_id) -> Relation: """Safe way of retrieving a relation.""" @@ -879,12 +958,19 @@ def fetch_my_relation_field( if relation_data := self.fetch_my_relation_data([relation_id], [field], relation_name): return relation_data.get(relation_id, {}).get(field) - # Public methods - mandatory override - - @abstractmethod + @leader_only def update_relation_data(self, relation_id: int, data: dict) -> None: """Update the data within the relation.""" - raise NotImplementedError + relation_name = self.relation_name + relation = self.get_relation(relation_name, relation_id) + return self._update_relation_data(relation, data) + + @leader_only + def delete_relation_data(self, relation_id: int, fields: List[str]) -> None: + """Remove field from the relation.""" + relation_name = self.relation_name + relation = self.get_relation(relation_name, relation_id) + return self._delete_relation_data(relation, fields) # Base DataProvides and DataRequires @@ -910,36 +996,32 @@ def _diff(self, event: RelationChangedEvent) -> Diff: # Private methods handling secrets - @leader_only @juju_secrets_only def _add_relation_secret( - self, relation_id: int, content: Dict[str, str], group_mapping: SecretGroup + self, relation: Relation, content: Dict[str, str], group_mapping: SecretGroup ) -> Optional[Secret]: """Add a new Juju Secret that will be registered in the relation databag.""" - relation = self.get_relation(self.relation_name, relation_id) - secret_field = self._generate_secret_field_name(group_mapping) if relation.data[self.local_app].get(secret_field): - logging.error("Secret for relation %s already exists, not adding again", relation_id) + logging.error("Secret for relation %s already exists, not adding again", relation.id) return - label = self._generate_secret_label(self.relation_name, relation_id, group_mapping) + label = self._generate_secret_label(self.relation_name, relation.id, group_mapping) secret = self.secrets.add(label, content, relation) # According to lint we may not have a Secret ID if secret.meta and secret.meta.id: relation.data[self.local_app][secret_field] = secret.meta.id - @leader_only @juju_secrets_only def _update_relation_secret( - self, relation_id: int, content: Dict[str, str], group_mapping: SecretGroup + self, relation: Relation, content: Dict[str, str], group_mapping: SecretGroup ): """Update the contents of an existing Juju Secret, referred in the relation databag.""" - secret = self._get_relation_secret(relation_id, group_mapping) + secret = self._get_relation_secret(relation.id, group_mapping) if not secret: - logging.error("Can't update secret for relation %s", relation_id) + logging.error("Can't update secret for relation %s", relation.id) return old_content = secret.get_content() @@ -947,22 +1029,39 @@ def _update_relation_secret( full_content.update(content) secret.set_content(full_content) - @staticmethod - def _secret_content_grouped( - content: Dict[str, str], secret_fields: Set[str], group_mapping: SecretGroup - ) -> Dict[str, str]: - if group_mapping == SecretGroup.EXTRA: - return { - k: v - for k, v in content.items() - if k in secret_fields and k not in SECRET_LABEL_MAP.keys() - } + def _add_or_update_relation_secrets( + self, + relation: Relation, + group: SecretGroup, + secret_fields: Set[str], + data: Dict[str, str], + ) -> None: + secret_content = self._secret_content_grouped(data, secret_fields, group) + if self._get_relation_secret(relation.id, group): + self._update_relation_secret(relation, secret_content, group) + else: + self._add_relation_secret(relation, secret_content, group) - return { - k: v - for k, v in content.items() - if k in secret_fields and SECRET_LABEL_MAP.get(k) == group_mapping - } + @juju_secrets_only + def _delete_relation_secret( + self, relation: Relation, group: SecretGroup, secret_fields: List[str], fields: List[str] + ): + """Update the contents of an existing Juju Secret, referred in the relation databag.""" + secret = self._get_relation_secret(relation.id, group) + + if not secret: + logging.error("Can't update secret for relation %s", relation.id) + return + + old_content = secret.get_content() + new_content = copy.deepcopy(old_content) + for field in fields: + new_content.pop(field) + secret.set_content(new_content) + + if not new_content: + field = self._generate_secret_field_name(group) + relation.data[self.local_app].pop(field) # Mandatory internal overrides @@ -1004,45 +1103,42 @@ def _fetch_my_specific_relation_data( """Fetching our own relation data.""" secret_fields = None if relation.app: - secret_fields = get_encoded_field(relation, relation.app, REQ_SECRET_FIELDS) + secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) return self._fetch_relation_data_with_secrets( self.local_app, - secret_fields if isinstance(secret_fields, list) else None, + secret_fields, relation, fields, ) - # Public methods -- mandatory overrides - - @leader_only - def update_relation_data(self, relation_id: int, fields: Dict[str, str]) -> None: + def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: """Set values for fields not caring whether it's a secret or not.""" - relation = self.get_relation(self.relation_name, relation_id) - + req_secret_fields = [] if relation.app: - relation_secret_fields = get_encoded_field(relation, relation.app, REQ_SECRET_FIELDS) - else: - relation_secret_fields = [] + req_secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) - normal_fields = list(fields) - if relation_secret_fields and self.secrets_enabled: - normal_fields = set(fields.keys()) - set(relation_secret_fields) - secret_fields = set(fields.keys()) - set(normal_fields) + _, normal_fields = self._process_secret_fields( + relation, + req_secret_fields, + list(data), + self._add_or_update_relation_secrets, + data=data, + ) - secret_fieldnames_grouped = self._group_secret_fields(list(secret_fields)) + normal_content = {k: v for k, v in data.items() if k in normal_fields} + self._update_relation_data_without_secrets(self.local_app, relation, normal_content) - for group in secret_fieldnames_grouped: - secret_content = self._secret_content_grouped(fields, secret_fields, group) - if self._get_relation_secret(relation_id, group): - self._update_relation_secret(relation_id, secret_content, group) - else: - self._add_relation_secret(relation_id, secret_content, group) - - normal_content = {k: v for k, v in fields.items() if k in normal_fields} - relation.data[self.local_app].update( # pyright: ignore [reportGeneralTypeIssues] - normal_content + def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: + """Delete fields from the Relation not caring whether it's a secret or not.""" + req_secret_fields = [] + if relation.app: + req_secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + + _, normal_fields = self._process_secret_fields( + relation, req_secret_fields, fields, self._delete_relation_secret, fields=fields ) + self._delete_relation_data_without_secrets(self.local_app, relation, list(normal_fields)) # Public methods - "native" @@ -1245,26 +1341,32 @@ def _fetch_my_specific_relation_data(self, relation, fields: Optional[List[str]] """Fetching our own relation data.""" return self._fetch_relation_data_without_secrets(self.local_app, relation, fields) - # Public methods -- mandatory overrides - @leader_only - def update_relation_data(self, relation_id: int, data: dict) -> None: + def _update_relation_data(self, relation: Relation, data: dict) -> None: """Updates a set of key-value pairs in the relation. This function writes in the application data bag, therefore, only the leader unit can call it. Args: - relation_id: the identifier for a particular relation. + relation: the particular relation. data: dict containing the key-value pairs that should be updated in the relation. """ - if any(self._is_secret_field(key) for key in data.keys()): - raise SecretsIllegalUpdateError("Requires side can't update secrets.") + return self._update_relation_data_without_secrets(self.local_app, relation, data) - relation = self.charm.model.get_relation(self.relation_name, relation_id) - if relation: - relation.data[self.local_app].update(data) + @leader_only + def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: + """Deletes a set of fields from the relation. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation: the particular relation. + fields: list containing the field names that should be removed from the relation. + """ + return self._delete_relation_data_without_secrets(self.local_app, relation, fields) # General events