From 9848bb8a7d9d0afe004c92638ee465b4b01e96be Mon Sep 17 00:00:00 2001 From: Dragomir Penev <6687393+dragomirp@users.noreply.github.com> Date: Sat, 21 Oct 2023 01:38:15 +0300 Subject: [PATCH] [DPE-1770] Minor version upgrades (#129) * Boilerplate * Fix scheduled tests * Happy scenario * Integration tests * Unit test * Switch to testing profile * Tenacity instead of defer * Tweak unit tests * Patch version bump * Fix CI and bump libs * Bump agent * Update tox.ini Co-authored-by: Marcelo Henrique Neppel --------- Co-authored-by: Marcelo Henrique Neppel --- .github/workflows/ci.yaml | 7 +- actions.yaml | 6 + .../data_platform_libs/v0/data_interfaces.py | 6 +- lib/charms/data_platform_libs/v0/upgrade.py | 1078 +++++++++++++++++ lib/charms/postgresql_k8s/v0/postgresql.py | 9 +- metadata.yaml | 2 + poetry.lock | 33 +- pyproject.toml | 1 + requirements.txt | 1 + src/charm.py | 80 +- src/dependency.json | 14 + src/relations/peers.py | 7 +- src/upgrade.py | 111 ++ tests/integration/helpers/helpers.py | 2 +- .../test_pgbouncer_provider.py | 2 + tests/integration/relations/test_peers.py | 4 +- tests/integration/test_charm.py | 1 + tests/integration/test_upgrade.py | 99 ++ tests/unit/test_charm.py | 5 +- tests/unit/test_upgrade.py | 138 +++ tox.ini | 11 + 21 files changed, 1551 insertions(+), 66 deletions(-) create mode 100644 actions.yaml create mode 100644 lib/charms/data_platform_libs/v0/upgrade.py create mode 100644 src/dependency.json create mode 100644 src/upgrade.py create mode 100644 tests/integration/test_upgrade.py create mode 100644 tests/unit/test_upgrade.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1c3454b5e..a7ab9f5ec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -78,6 +78,7 @@ jobs: - legacy-client-relation-integration - legacy-client-relation-integration-admin - scaling-integration + - upgrade-integration agent-versions: - "2.9.45" # renovate: latest juju 2 - "3.1.6" # renovate: latest juju 3 @@ -85,7 +86,7 @@ jobs: - false include: - tox-environments: client-relation-integration - agent-versions: "2.9.44" # renovate: latest juju 2 + agent-versions: "2.9.45" # renovate: latest juju 2 free-disk-space: true - tox-environments: client-relation-integration agent-versions: "3.1.6" # renovate: latest juju 3 @@ -135,9 +136,9 @@ jobs: echo "mark_expression=" >> $GITHUB_OUTPUT else echo Skipping unstable tests - echo "mark_expression=not unstable" >> $GITHUB_OUTPUT + echo "mark_expression=and not unstable" >> $GITHUB_OUTPUT fi - name: Run integration tests - run: tox run -e ${{ matrix.tox-environments }}-${{ env.libjuju }} -- -m 'not not${{ env.libjuju }} and ${{ steps.select-tests.outputs.mark_expression }}' + run: tox run -e ${{ matrix.tox-environments }}-${{ env.libjuju }} -- -m 'not not${{ env.libjuju }} ${{ steps.select-tests.outputs.mark_expression }}' --keep-models env: CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} diff --git a/actions.yaml b/actions.yaml new file mode 100644 index 000000000..996aaff87 --- /dev/null +++ b/actions.yaml @@ -0,0 +1,6 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +pre-upgrade-check: + description: Run necessary pre-upgrade checks before executing a charm upgrade. + diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index 2624dd4d6..9071655a8 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -320,7 +320,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 19 +LIBPATCH = 20 PYDEPS = ["ops>=2.0.0"] @@ -1674,6 +1674,10 @@ def _assign_relation_alias(self, relation_id: int) -> None: if relation: relation.data[self.local_unit].update({"alias": available_aliases[0]}) + # We need to set relation alias also on the application level so, + # it will be accessible in show-unit juju command, executed for a consumer application unit + self.update_relation_data(relation_id, {"alias": available_aliases[0]}) + def _emit_aliased_event(self, event: RelationChangedEvent, event_name: str) -> None: """Emit an aliased event to a particular relation if it has an alias. diff --git a/lib/charms/data_platform_libs/v0/upgrade.py b/lib/charms/data_platform_libs/v0/upgrade.py new file mode 100644 index 000000000..4ee2e9ff8 --- /dev/null +++ b/lib/charms/data_platform_libs/v0/upgrade.py @@ -0,0 +1,1078 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +r"""Library to manage in-place upgrades for charms running on VMs and K8s. + +This library contains handlers for `upgrade` relation events used to coordinate +between units in an application during a `juju refresh`, as well as `Pydantic` models +for instantiating, validating and comparing dependencies. + +An upgrade on VMs is initiated with the command `juju refresh`. Once executed, the following +events are emitted to each unit at random: + - `upgrade-charm` + - `config-changed` + - `leader-settings-changed` - Non-leader only + +Charm authors can implement the classes defined in this library to streamline the process of +coordinating which unit updates when, achieved through updating of unit-data `state` throughout. + +At a high-level, the upgrade steps are as follows: + - Run pre-checks on the cluster to confirm it is safe to upgrade + - Create stack of unit.ids, to serve as the upgrade order (generally workload leader is last) + - Start the upgrade by issuing a Juju CLI command + - The unit at the top of the stack gets permission to upgrade + - The unit handles the upgrade and restarts their service + - Repeat, until all units have restarted + +### Usage by charm authors + +#### `upgrade` relation + +Charm authors must implement an additional peer-relation. + +As this library uses relation data exchanged between units to coordinate, charm authors +need to add a new relation interface. The relation name does not matter. + +`metadata.yaml` +```yaml +peers: + upgrade: + interface: upgrade +``` + +#### Dependencies JSON/Dict + +Charm authors must implement a dict object tracking current charm versions, requirements + upgradability. + +Many workload versions may be incompatible with older/newer versions. This same idea also can apply to +charm or snap versions. Workloads with required related applications (e.g Kafka + ZooKeeper) also need to +ensure their versions are compatible during an upgrade, to avoid cluster failure. + +As such, it is necessasry to freeze any dependencies within each published charm. An example of this could +be creating a `DEPENDENCIES` dict within the charm code, with the following structure: + +`src/literals.py` +```python +DEPENDENCIES = { + "kafka_charm": { + "dependencies": {"zookeeper": ">50"}, + "name": "kafka", + "upgrade_supported": ">90", + "version": "100", + }, + "kafka_service": { + "dependencies": {"zookeeper": "^3"}, + "name": "kafka", + "upgrade_supported": ">=0.8", + "version": "3.3.2", + }, +} +``` + +The first-level key names are arbitrary labels for tracking what those versions+dependencies are for. +The `dependencies` second-level values are a key-value map of any required external applications, + and the versions this packaged charm can support. +The `upgrade_suppported` second-level values are requirements from which an in-place upgrade can be + supported by the charm. +The `version` second-level values correspond to the current version of this packaged charm. + +Any requirements comply with [`poetry`'s dependency specifications](https://python-poetry.org/docs/dependency-specification/#caret-requirements). + +### Dependency Model + +Charm authors must implement their own class inheriting from `DependencyModel`. + +Using a `Pydantic` model to instantiate the aforementioned `DEPENDENCIES` dict gives stronger type safety and additional +layers of validation. + +Implementation just needs to ensure that the top-level key names from `DEPENDENCIES` are defined as attributed in the model. + +`src/upgrade.py` +```python +from pydantic import BaseModel + +class KafkaDependenciesModel(BaseModel): + kafka_charm: DependencyModel + kafka_service: DependencyModel +``` + +### Overrides for `DataUpgrade` + +Charm authors must define their own class, inheriting from `DataUpgrade`, overriding all required `abstractmethod`s. + +```python +class ZooKeeperUpgrade(DataUpgrade): + def __init__(self, charm: "ZooKeeperUpgrade", **kwargs): + super().__init__(charm, **kwargs) + self.charm = charm +``` + +#### Implementation of `pre_upgrade_check()` + +Before upgrading a cluster, it's a good idea to check that it is stable and healthy before permitting it. +Here, charm authors can validate upgrade safety through API calls, relation-data checks, etc. +If any of these checks fail, raise `ClusterNotReadyError`. + +```python + @override + def pre_upgrade_check(self) -> None: + default_message = "Pre-upgrade check failed and cannot safely upgrade" + try: + if not self.client.members_broadcasting or not len(self.client.server_members) == len( + self.charm.cluster.peer_units + ): + raise ClusterNotReadyError( + message=default_message, + cause="Not all application units are connected and broadcasting in the quorum", + ) + + if self.client.members_syncing: + raise ClusterNotReadyError( + message=default_message, cause="Some quorum members are syncing data" + ) + + if not self.charm.cluster.stable: + raise ClusterNotReadyError( + message=default_message, cause="Charm has not finished initialising" + ) + + except QuorumLeaderNotFoundError: + raise ClusterNotReadyError(message=default_message, cause="Quorum leader not found") + except ConnectionClosedError: + raise ClusterNotReadyError( + message=default_message, cause="Unable to connect to the cluster" + ) +``` + +#### Implementation of `build_upgrade_stack()` - VM ONLY + +Oftentimes, it is necessary to ensure that the workload leader is the last unit to upgrade, +to ensure high-availability during the upgrade process. +Here, charm authors can create a LIFO stack of unit.ids, represented as a list of unit.id strings, +with the leader unit being at i[0]. + +```python +@override +def build_upgrade_stack(self) -> list[int]: + upgrade_stack = [] + for unit in self.charm.cluster.peer_units: + config = self.charm.cluster.unit_config(unit=unit) + + # upgrade quorum leader last + if config["host"] == self.client.leader: + upgrade_stack.insert(0, int(config["unit_id"])) + else: + upgrade_stack.append(int(config["unit_id"])) + + return upgrade_stack +``` + +#### Implementation of `_on_upgrade_granted()` + +On relation-changed events, each unit will check the current upgrade-stack persisted to relation data. +If that unit is at the top of the stack, it will emit an `upgrade-granted` event, which must be handled. +Here, workloads can be re-installed with new versions, checks can be made, data synced etc. +If the new unit successfully rejoined the cluster, call `set_unit_completed()`. +If the new unit failed to rejoin the cluster, call `set_unit_failed()`. + +NOTE - It is essential here to manually call `on_upgrade_changed` if the unit is the current leader. +This ensures that the leader gets it's own relation-changed event, and updates the upgrade-stack for +other units to follow suit. + +```python +@override +def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None: + self.charm.snap.stop_snap_service() + + if not self.charm.snap.install(): + logger.error("Unable to install ZooKeeper Snap") + self.set_unit_failed() + return None + + logger.info(f"{self.charm.unit.name} upgrading service...") + self.charm.snap.restart_snap_service() + + try: + logger.debug("Running post-upgrade check...") + self.pre_upgrade_check() + + logger.debug("Marking unit completed...") + self.set_unit_completed() + + # ensures leader gets it's own relation-changed when it upgrades + if self.charm.unit.is_leader(): + logger.debug("Re-emitting upgrade-changed on leader...") + self.on_upgrade_changed(event) + + except ClusterNotReadyError as e: + logger.error(e.cause) + self.set_unit_failed() +``` + +#### Implementation of `log_rollback_instructions()` + +If the upgrade fails, manual intervention may be required for cluster recovery. +Here, charm authors can log out any necessary steps to take to recover from a failed upgrade. +When a unit fails, this library will automatically log out this message. + +```python +@override +def log_rollback_instructions(self) -> None: + logger.error("Upgrade failed. Please run `juju refresh` to previous version.") +``` + +### Instantiating in the charm and deferring events + +Charm authors must add a class attribute for the child class of `DataUpgrade` in the main charm. +They must also ensure that any non-upgrade related events that may be unsafe to handle during +an upgrade, are deferred if the unit is not in the `idle` state - i.e not currently upgrading. + +```python +class ZooKeeperCharm(CharmBase): + def __init__(self, *args): + super().__init__(*args) + self.upgrade = ZooKeeperUpgrade( + self, + relation_name = "upgrade", + substrate = "vm", + dependency_model=ZooKeeperDependencyModel( + **DEPENDENCIES + ), + ) + + def restart(self, event) -> None: + if not self.upgrade.state == "idle": + event.defer() + return None + + self.restart_snap_service() +``` +""" + +import json +import logging +from abc import ABC, abstractmethod +from typing import Dict, List, Literal, Optional, Set, Tuple + +import poetry.core.constraints.version as poetry_version +from ops.charm import ( + ActionEvent, + CharmBase, + CharmEvents, + RelationCreatedEvent, + UpgradeCharmEvent, +) +from ops.framework import EventBase, EventSource, Object +from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus, Relation, Unit, WaitingStatus +from pydantic import BaseModel, root_validator, validator + +# The unique Charmhub library identifier, never change it +LIBID = "156258aefb79435a93d933409a8c8684" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 15 + +PYDEPS = ["pydantic>=1.10,<2", "poetry-core"] + +logger = logging.getLogger(__name__) + +# --- DEPENDENCY RESOLUTION FUNCTIONS --- + + +def verify_requirements(version: str, requirement: str) -> bool: + """Verifies a specified version against defined constraint. + + Supports Poetry version constraints + https://python-poetry.org/docs/dependency-specification/#version-constraints + + Args: + version: the version currently in use + requirement: Poetry version constraint + + Returns: + True if `version` meets defined `requirement`. Otherwise False + """ + return poetry_version.parse_constraint(requirement).allows( + poetry_version.Version.parse(version) + ) + + +# --- DEPENDENCY MODEL TYPES --- + + +class DependencyModel(BaseModel): + """Manager for a single dependency. + + To be used as part of another model representing a collection of arbitrary dependencies. + + Example:: + + class KafkaDependenciesModel(BaseModel): + kafka_charm: DependencyModel + kafka_service: DependencyModel + + deps = { + "kafka_charm": { + "dependencies": {"zookeeper": ">5"}, + "name": "kafka", + "upgrade_supported": ">5", + "version": "10", + }, + "kafka_service": { + "dependencies": {"zookeeper": "^3.6"}, + "name": "kafka", + "upgrade_supported": "~3.3", + "version": "3.3.2", + }, + } + + model = KafkaDependenciesModel(**deps) # loading dict in to model + + print(model.dict()) # exporting back validated deps + """ + + dependencies: Dict[str, str] + name: str + upgrade_supported: str + version: str + + @validator("dependencies", "upgrade_supported", each_item=True) + @classmethod + def dependencies_validator(cls, value): + """Validates version constraint.""" + if isinstance(value, dict): + deps = value.values() + else: + deps = [value] + + for dep in deps: + poetry_version.parse_constraint(dep) + + return value + + @root_validator(skip_on_failure=True) + @classmethod + def version_upgrade_supported_validator(cls, values): + """Validates specified `version` meets `upgrade_supported` requirement.""" + if not verify_requirements( + version=values.get("version"), requirement=values.get("upgrade_supported") + ): + raise ValueError( + f"upgrade_supported value {values.get('upgrade_supported')} greater than version value {values.get('version')} for {values.get('name')}." + ) + + return values + + def can_upgrade(self, dependency: "DependencyModel") -> bool: + """Compares two instances of :class:`DependencyModel` for upgradability. + + Args: + dependency: a dependency model to compare this model against + + Returns: + True if current model can upgrade from dependent model. Otherwise False + """ + return verify_requirements(version=self.version, requirement=dependency.upgrade_supported) + + +# --- CUSTOM EXCEPTIONS --- + + +class UpgradeError(Exception): + """Base class for upgrade related exceptions in the module.""" + + def __init__(self, message: str, cause: Optional[str], resolution: Optional[str]): + super().__init__(message) + self.message = message + self.cause = cause or "" + self.resolution = resolution or "" + + def __repr__(self): + """Representation of the UpgradeError class.""" + return f"{type(self).__module__}.{type(self).__name__} - {str(vars(self))}" + + def __str__(self): + """String representation of the UpgradeError class.""" + return repr(self) + + +class ClusterNotReadyError(UpgradeError): + """Exception flagging that the cluster is not ready to start upgrading. + + For example, if the cluster fails :class:`DataUpgrade._on_pre_upgrade_check_action` + + Args: + message: string message to be logged out + cause: short human-readable description of the cause of the error + resolution: short human-readable instructions for manual error resolution (optional) + """ + + def __init__(self, message: str, cause: str, resolution: Optional[str] = None): + super().__init__(message, cause=cause, resolution=resolution) + + +class KubernetesClientError(UpgradeError): + """Exception flagging that a call to Kubernetes API failed. + + For example, if the cluster fails :class:`DataUpgrade._set_rolling_update_partition` + + Args: + message: string message to be logged out + cause: short human-readable description of the cause of the error + resolution: short human-readable instructions for manual error resolution (optional) + """ + + def __init__(self, message: str, cause: str, resolution: Optional[str] = None): + super().__init__(message, cause=cause, resolution=resolution) + + +class VersionError(UpgradeError): + """Exception flagging that the old `version` fails to meet the new `upgrade_supported`s. + + For example, upgrades from version `2.x` --> `4.x`, + but `4.x` only supports upgrading from `3.x` onwards + + Args: + message: string message to be logged out + cause: short human-readable description of the cause of the error + resolution: short human-readable instructions for manual solutions to the error (optional) + """ + + def __init__(self, message: str, cause: str, resolution: Optional[str] = None): + super().__init__(message, cause=cause, resolution=resolution) + + +class DependencyError(UpgradeError): + """Exception flagging that some new `dependency` is not being met. + + For example, new version requires related App version `2.x`, but currently is `1.x` + + Args: + message: string message to be logged out + cause: short human-readable description of the cause of the error + resolution: short human-readable instructions for manual solutions to the error (optional) + """ + + def __init__(self, message: str, cause: str, resolution: Optional[str] = None): + super().__init__(message, cause=cause, resolution=resolution) + + +# --- CUSTOM EVENTS --- + + +class UpgradeGrantedEvent(EventBase): + """Used to tell units that they can process an upgrade.""" + + +class UpgradeFinishedEvent(EventBase): + """Used to tell units that they finished the upgrade.""" + + +class UpgradeEvents(CharmEvents): + """Upgrade events. + + This class defines the events that the lib can emit. + """ + + upgrade_granted = EventSource(UpgradeGrantedEvent) + upgrade_finished = EventSource(UpgradeFinishedEvent) + + +# --- EVENT HANDLER --- + + +class DataUpgrade(Object, ABC): + """Manages `upgrade` relation operations for in-place upgrades.""" + + STATES = ["recovery", "failed", "idle", "ready", "upgrading", "completed"] + + on = UpgradeEvents() # pyright: ignore [reportGeneralTypeIssues] + + def __init__( + self, + charm: CharmBase, + dependency_model: BaseModel, + relation_name: str = "upgrade", + substrate: Literal["vm", "k8s"] = "vm", + ): + super().__init__(charm, relation_name) + self.charm = charm + self.dependency_model = dependency_model + self.relation_name = relation_name + self.substrate = substrate + self._upgrade_stack = None + + # events + self.framework.observe( + self.charm.on[relation_name].relation_created, self._on_upgrade_created + ) + self.framework.observe( + self.charm.on[relation_name].relation_changed, self.on_upgrade_changed + ) + self.framework.observe(self.charm.on.upgrade_charm, self._on_upgrade_charm) + self.framework.observe(getattr(self.on, "upgrade_granted"), self._on_upgrade_granted) + self.framework.observe(getattr(self.on, "upgrade_finished"), self._on_upgrade_finished) + + # actions + self.framework.observe( + getattr(self.charm.on, "pre_upgrade_check_action"), self._on_pre_upgrade_check_action + ) + if self.substrate == "k8s": + self.framework.observe( + getattr(self.charm.on, "resume_upgrade_action"), self._on_resume_upgrade_action + ) + + @property + def peer_relation(self) -> Optional[Relation]: + """The upgrade peer relation.""" + return self.charm.model.get_relation(self.relation_name) + + @property + def app_units(self) -> Set[Unit]: + """The peer-related units in the application.""" + if not self.peer_relation: + return set() + + return set([self.charm.unit] + list(self.peer_relation.units)) + + @property + def state(self) -> Optional[str]: + """The unit state from the upgrade peer relation.""" + if not self.peer_relation: + return None + + return self.peer_relation.data[self.charm.unit].get("state", None) + + @property + def stored_dependencies(self) -> Optional[BaseModel]: + """The application dependencies from the upgrade peer relation.""" + if not self.peer_relation: + return None + + if not (deps := self.peer_relation.data[self.charm.app].get("dependencies", "")): + return None + + return type(self.dependency_model)(**json.loads(deps)) + + @property + def upgrade_stack(self) -> Optional[List[int]]: + """Gets the upgrade stack from the upgrade peer relation. + + Unit.ids are ordered Last-In-First-Out (LIFO). + i.e unit.id at index `-1` is the first unit to upgrade. + unit.id at index `0` is the last unit to upgrade. + + Returns: + List of integer unit.ids, ordered in upgrade order in a stack + """ + if not self.peer_relation: + return None + + # lazy-load + if self._upgrade_stack is None: + self._upgrade_stack = ( + json.loads(self.peer_relation.data[self.charm.app].get("upgrade-stack", "[]")) + or None + ) + + return self._upgrade_stack + + @upgrade_stack.setter + def upgrade_stack(self, stack: List[int]) -> None: + """Sets the upgrade stack to the upgrade peer relation. + + Unit.ids are ordered Last-In-First-Out (LIFO). + i.e unit.id at index `-1` is the first unit to upgrade. + unit.id at index `0` is the last unit to upgrade. + """ + if not self.peer_relation: + return + + self.peer_relation.data[self.charm.app].update({"upgrade-stack": json.dumps(stack)}) + self._upgrade_stack = stack + + @property + def unit_states(self) -> list: + """Current upgrade state for all units. + + Returns: + Unsorted list of upgrade states for all units. + """ + if not self.peer_relation: + return [] + + return [self.peer_relation.data[unit].get("state", "") for unit in self.app_units] + + @property + def cluster_state(self) -> Optional[str]: + """Current upgrade state for cluster units. + + Determined from :class:`DataUpgrade.STATE`, taking the lowest ordinal unit state. + + For example, if units in have states: `["ready", "upgrading", "completed"]`, + the overall state for the cluster is `ready`. + + Returns: + String of upgrade state from the furthest behind unit. + """ + if not self.unit_states: + return None + + try: + return sorted(self.unit_states, key=self.STATES.index)[0] + except (ValueError, KeyError): + return None + + @property + def idle(self) -> Optional[bool]: + """Flag for whether the cluster is in an idle upgrade state. + + Returns: + True if all application units in idle state. Otherwise False + """ + return set(self.unit_states) == {"idle"} + + @abstractmethod + def pre_upgrade_check(self) -> None: + """Runs necessary checks validating the cluster is in a healthy state to upgrade. + + Called by all units during :meth:`_on_pre_upgrade_check_action`. + + Raises: + :class:`ClusterNotReadyError`: if cluster is not ready to upgrade + """ + pass + + def build_upgrade_stack(self) -> List[int]: + """Builds ordered iterable of all application unit.ids to upgrade in. + + Called by leader unit during :meth:`_on_pre_upgrade_check_action`. + + Returns: + Iterable of integer unit.ids, LIFO ordered in upgrade order + i.e `[5, 2, 4, 1, 3]`, unit `3` upgrades first, `5` upgrades last + """ + # don't raise if k8s substrate, uses default statefulset order + if self.substrate == "k8s": + return [] + + raise NotImplementedError + + @abstractmethod + def log_rollback_instructions(self) -> None: + """Sets charm state and logs out rollback instructions. + + Called by all units when `state=failed` found during :meth:`_on_upgrade_changed`. + """ + pass + + def _repair_upgrade_stack(self) -> None: + """Ensures completed units are re-added to the upgrade-stack after failure.""" + # need to update the stack as it was not refreshed by rollback run of pre-upgrade-check + # avoids difficult health check implementation by charm-authors needing to exclude dead units + + # if the first unit in the stack fails, the stack will be the same length as units + # i.e this block not ran + if ( + self.cluster_state in ["failed", "recovery"] + and self.upgrade_stack + and len(self.upgrade_stack) != len(self.app_units) + and self.charm.unit.is_leader() + ): + new_stack = self.upgrade_stack + for unit in self.app_units: + unit_id = int(unit.name.split("/")[1]) + + # if a unit fails, it rolls back first + if unit_id not in new_stack: + new_stack.insert(-1, unit_id) + logger.debug(f"Inserted {unit_id} in to upgrade-stack - {new_stack}") + + self.upgrade_stack = new_stack + + def set_unit_failed(self, cause: Optional[str] = None) -> None: + """Sets unit `state=failed` to the upgrade peer data. + + Args: + cause: short description of cause of failure + """ + if not self.peer_relation: + return None + + # needed to refresh the stack + # now leader pulls a fresh stack from newly updated relation data + if self.charm.unit.is_leader(): + self._upgrade_stack = None + + self.charm.unit.status = BlockedStatus(cause if cause else "") + self.peer_relation.data[self.charm.unit].update({"state": "failed"}) + self.log_rollback_instructions() + + def set_unit_completed(self) -> None: + """Sets unit `state=completed` to the upgrade peer data.""" + if not self.peer_relation: + return None + + # needed to refresh the stack + # now leader pulls a fresh stack from newly updated relation data + if self.charm.unit.is_leader(): + self._upgrade_stack = None + + self.charm.unit.status = MaintenanceStatus("upgrade completed") + self.peer_relation.data[self.charm.unit].update({"state": "completed"}) + + # Emit upgrade_finished event to run unit's post upgrade operations. + if self.substrate == "k8s": + logger.debug( + f"{self.charm.unit.name} has completed the upgrade, emitting `upgrade_finished` event..." + ) + getattr(self.on, "upgrade_finished").emit() + + def _on_upgrade_created(self, event: RelationCreatedEvent) -> None: + """Handler for `upgrade-relation-created` events.""" + if not self.peer_relation: + event.defer() + return + + # setting initial idle state needed to avoid execution on upgrade-changed events + self.peer_relation.data[self.charm.unit].update({"state": "idle"}) + + if self.charm.unit.is_leader(): + logger.debug("Persisting dependencies to upgrade relation data...") + self.peer_relation.data[self.charm.app].update( + {"dependencies": json.dumps(self.dependency_model.dict())} + ) + + def _on_pre_upgrade_check_action(self, event: ActionEvent) -> None: + """Handler for `pre-upgrade-check-action` events.""" + if not self.peer_relation: + event.fail(message="Could not find upgrade relation.") + return + + if not self.charm.unit.is_leader(): + event.fail(message="Action must be ran on the Juju leader.") + return + + if self.cluster_state == "failed": + logger.info("Entering recovery state for rolling-back to previous version...") + self._repair_upgrade_stack() + self.charm.unit.status = BlockedStatus("ready to rollback application") + self.peer_relation.data[self.charm.unit].update({"state": "recovery"}) + return + + # checking if upgrade in progress + if self.cluster_state != "idle": + event.fail("Cannot run pre-upgrade checks, cluster already upgrading.") + return + + try: + logger.info("Running pre-upgrade-check...") + self.pre_upgrade_check() + + if self.substrate == "k8s": + logger.info("Building upgrade-stack for K8s...") + built_upgrade_stack = sorted( + [int(unit.name.split("/")[1]) for unit in self.app_units] + ) + else: + logger.info("Building upgrade-stack for VMs...") + built_upgrade_stack = self.build_upgrade_stack() + + logger.debug(f"Built upgrade stack of {built_upgrade_stack}") + + except ClusterNotReadyError as e: + logger.error(e) + event.fail(message=e.message) + return + except Exception as e: + logger.error(e) + event.fail(message="Unknown error found.") + return + + logger.info("Setting upgrade-stack to relation data...") + self.upgrade_stack = built_upgrade_stack + + def _on_resume_upgrade_action(self, event: ActionEvent) -> None: + """Handle resume upgrade action. + + Continue the upgrade by setting the partition to the next unit. + """ + if not self.peer_relation: + event.fail(message="Could not find upgrade relation.") + return + + if not self.charm.unit.is_leader(): + event.fail(message="Action must be ran on the Juju leader.") + return + + if not self.upgrade_stack: + event.fail(message="Nothing to resume, upgrade stack unset.") + return + + # Check whether this is being run after juju refresh was called + # (the size of the upgrade stack should match the number of total + # unit minus one). + if len(self.upgrade_stack) != len(self.peer_relation.units): + event.fail(message="Upgrade can be resumed only once after juju refresh is called.") + return + + try: + next_partition = self.upgrade_stack[-1] + self._set_rolling_update_partition(partition=next_partition) + event.set_results({"message": f"Upgrade will resume on unit {next_partition}"}) + except KubernetesClientError: + event.fail(message="Cannot set rolling update partition.") + + def _upgrade_supported_check(self) -> None: + """Checks if previous versions can be upgraded to new versions. + + Raises: + :class:`VersionError` if upgrading to existing `version` is not supported + """ + keys = self.dependency_model.__fields__.keys() + + compatible = True + incompatibilities: List[Tuple[str, str, str, str]] = [] + for key in keys: + old_dep: DependencyModel = getattr(self.stored_dependencies, key) + new_dep: DependencyModel = getattr(self.dependency_model, key) + + if not old_dep.can_upgrade(dependency=new_dep): + compatible = False + incompatibilities.append( + (key, old_dep.version, new_dep.version, new_dep.upgrade_supported) + ) + + base_message = "Versions incompatible" + base_cause = "Upgrades only supported for specific versions" + if not compatible: + for incompat in incompatibilities: + base_message += ( + f", {incompat[0]} {incompat[1]} can not be upgraded to {incompat[2]}" + ) + base_cause += f", {incompat[0]} versions satisfying requirement {incompat[3]}" + + raise VersionError( + message=base_message, + cause=base_cause, + ) + + def _on_upgrade_charm(self, event: UpgradeCharmEvent) -> None: + """Handler for `upgrade-charm` events.""" + # defer if not all units have pre-upgraded + if not self.peer_relation: + event.defer() + return + + if not self.upgrade_stack: + logger.error("Cluster upgrade failed, ensure pre-upgrade checks are ran first.") + return + + if self.substrate == "vm": + # for VM run version checks on leader only + if self.charm.unit.is_leader(): + try: + self._upgrade_supported_check() + except VersionError as e: # not ready if not passed check + logger.error(e) + self.set_unit_failed() + return + self.charm.unit.status = WaitingStatus("other units upgrading first...") + self.peer_relation.data[self.charm.unit].update({"state": "ready"}) + + if self.charm.app.planned_units() == 1: + # single unit upgrade, emit upgrade_granted event right away + getattr(self.on, "upgrade_granted").emit() + + else: + # for k8s run version checks only on highest ordinal unit + if ( + self.charm.unit.name + == f"{self.charm.app.name}/{self.charm.app.planned_units() -1}" + ): + try: + self._upgrade_supported_check() + except VersionError as e: # not ready if not passed check + logger.error(e) + self.set_unit_failed() + return + # On K8s an unit that receives the upgrade-charm event is upgrading + self.charm.unit.status = MaintenanceStatus("upgrading unit") + self.peer_relation.data[self.charm.unit].update({"state": "upgrading"}) + + def on_upgrade_changed(self, event: EventBase) -> None: + """Handler for `upgrade-relation-changed` events.""" + if not self.peer_relation: + return + + # if any other unit failed, don't continue with upgrade + if self.cluster_state == "failed": + logger.debug("Cluster failed to upgrade, exiting...") + return + + if self.substrate == "vm" and self.cluster_state == "recovery": + # Only defer for vm, that will set unit states to "ready" on upgrade-charm + # on k8s only the upgrading unit will receive the upgrade-charm event + # and deferring will prevent the upgrade stack from being popped + logger.debug("Cluster in recovery, deferring...") + event.defer() + return + + # if all units completed, mark as complete + if not self.upgrade_stack: + if self.state == "completed" and self.cluster_state in ["idle", "completed"]: + logger.info("All units completed upgrade, setting idle upgrade state...") + self.charm.unit.status = ActiveStatus() + self.peer_relation.data[self.charm.unit].update({"state": "idle"}) + + if self.charm.unit.is_leader(): + logger.debug("Persisting new dependencies to upgrade relation data...") + self.peer_relation.data[self.charm.app].update( + {"dependencies": json.dumps(self.dependency_model.dict())} + ) + return + + if self.cluster_state == "idle": + logger.debug("upgrade-changed event handled before pre-checks, exiting...") + return + + logger.debug("Did not find upgrade-stack or completed cluster state, skipping...") + return + + # upgrade ongoing, set status for waiting units + if "upgrading" in self.unit_states and self.state in ["idle", "ready"]: + self.charm.unit.status = WaitingStatus("other units upgrading first...") + + # pop mutates the `upgrade_stack` attr + top_unit_id = self.upgrade_stack.pop() + top_unit = self.charm.model.get_unit(f"{self.charm.app.name}/{top_unit_id}") + top_state = self.peer_relation.data[top_unit].get("state") + + # if top of stack is completed, leader pops it + if self.charm.unit.is_leader() and top_state == "completed": + logger.debug(f"{top_unit} has finished upgrading, updating stack...") + + # writes the mutated attr back to rel data + self.peer_relation.data[self.charm.app].update( + {"upgrade-stack": json.dumps(self.upgrade_stack)} + ) + + # recurse on leader to ensure relation changed event not lost + # in case leader is next or the last unit to complete + self.on_upgrade_changed(event) + + # if unit top of stack and all units ready (i.e stack), emit granted event + if ( + self.charm.unit == top_unit + and top_state in ["ready", "upgrading"] + and self.cluster_state == "ready" + ): + logger.debug( + f"{top_unit.name} is next to upgrade, emitting `upgrade_granted` event and upgrading..." + ) + self.charm.unit.status = MaintenanceStatus("upgrading...") + self.peer_relation.data[self.charm.unit].update({"state": "upgrading"}) + + try: + getattr(self.on, "upgrade_granted").emit() + except DependencyError as e: + logger.error(e) + self.set_unit_failed() + return + + def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None: + """Handler for `upgrade-granted` events. + + Handlers of this event must meet the following: + - SHOULD check for related application deps from :class:`DataUpgrade.dependencies` + - MAY raise :class:`DependencyError` if dependency not met + - MUST update unit `state` after validating the success of the upgrade, calling one of: + - :class:`DataUpgrade.set_unit_failed` if the unit upgrade fails + - :class:`DataUpgrade.set_unit_completed` if the unit upgrade succeeds + - MUST call :class:`DataUpgarde.on_upgrade_changed` on exit so event not lost on leader + """ + # don't raise if k8s substrate, only return + if self.substrate == "k8s": + return + + raise NotImplementedError + + def _on_upgrade_finished(self, _) -> None: + """Handler for `upgrade-finished` events.""" + if self.substrate == "vm" or not self.peer_relation: + return + + # Emit the upgrade relation changed event in the leader to update the upgrade_stack. + if self.charm.unit.is_leader(): + self.charm.on[self.relation_name].relation_changed.emit( + self.model.get_relation(self.relation_name) + ) + + # This hook shouldn't run for the last unit (the first that is upgraded). For that unit it + # should be done through an action after the upgrade success on that unit is double-checked. + unit_number = int(self.charm.unit.name.split("/")[1]) + if unit_number == len(self.peer_relation.units): + logger.info( + f"{self.charm.unit.name} unit upgraded. Evaluate and run `resume-upgrade` action to continue upgrade" + ) + return + + # Also, the hook shouldn't run for the first unit (the last that is upgraded). + if unit_number == 0: + logger.info(f"{self.charm.unit.name} unit upgraded. Upgrade is complete") + return + + try: + # Use the unit number instead of the upgrade stack to avoid race conditions + # (i.e. the leader updates the upgrade stack after this hook runs). + next_partition = unit_number - 1 + logger.debug(f"Set rolling update partition to unit {next_partition}") + self._set_rolling_update_partition(partition=next_partition) + except KubernetesClientError: + logger.exception("Cannot set rolling update partition") + self.set_unit_failed() + self.log_rollback_instructions() + + def _set_rolling_update_partition(self, partition: int) -> None: + """Patch the StatefulSet's `spec.updateStrategy.rollingUpdate.partition`. + + Args: + partition: partition to set. + + K8s only. It should decrement the rolling update strategy partition by using a code + like the following: + + from lightkube.core.client import Client + from lightkube.core.exceptions import ApiError + from lightkube.resources.apps_v1 import StatefulSet + + try: + patch = {"spec": {"updateStrategy": {"rollingUpdate": {"partition": partition}}}} + Client().patch(StatefulSet, name=self.charm.model.app.name, namespace=self.charm.model.name, obj=patch) + logger.debug(f"Kubernetes StatefulSet partition set to {partition}") + except ApiError as e: + if e.status.code == 403: + cause = "`juju trust` needed" + else: + cause = str(e) + raise KubernetesClientError("Kubernetes StatefulSet patch failed", cause) + """ + if self.substrate == "vm": + return + + raise NotImplementedError diff --git a/lib/charms/postgresql_k8s/v0/postgresql.py b/lib/charms/postgresql_k8s/v0/postgresql.py index bfda780e8..3efcda0c4 100644 --- a/lib/charms/postgresql_k8s/v0/postgresql.py +++ b/lib/charms/postgresql_k8s/v0/postgresql.py @@ -32,7 +32,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 16 +LIBPATCH = 17 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles" @@ -117,12 +117,13 @@ def _connect_to_database( connection.autocommit = True return connection - def create_database(self, database: str, user: str) -> None: + def create_database(self, database: str, user: str, plugins: List[str] = []) -> None: """Creates a new database and grant privileges to a user on it. Args: database: database to be created. user: user that will have access to the database. + plugins: extensions to enable in the new database. """ try: connection = self._connect_to_database() @@ -170,6 +171,10 @@ def create_database(self, database: str, user: str) -> None: logger.error(f"Failed to create database: {e}") raise PostgreSQLCreateDatabaseError() + # Enable preset extensions + for plugin in plugins: + self.enable_disable_extension(plugin, True, database) + def create_user( self, user: str, password: str = None, admin: bool = False, extra_user_roles: str = None ) -> None: diff --git a/metadata.yaml b/metadata.yaml index 1144daddc..8c5f53aa7 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -27,6 +27,8 @@ series: peers: pgb-peers: interface: pgb_peers + upgrade: + interface: upgrade subordinate: true diff --git a/poetry.lock b/poetry.lock index 4e90e75b2..70d1d8da7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -777,16 +777,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1001,6 +991,17 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "poetry-core" +version = "1.7.0" +description = "Poetry PEP 517 Build Backend" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "poetry_core-1.7.0-py3-none-any.whl", hash = "sha256:38e174cdb00a84ee4a1cab66a378b435747f72414f5573bc18cfc3850a94df38"}, + {file = "poetry_core-1.7.0.tar.gz", hash = "sha256:8f679b83bd9c820082637beca1204124d5d2a786e4818da47ec8acefd0353b74"}, +] + [[package]] name = "prompt-toolkit" version = "3.0.39" @@ -1344,7 +1345,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1352,15 +1352,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1377,7 +1370,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1385,7 +1377,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1779,4 +1770,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8.10" -content-hash = "fd6108b07bfb536d151afabd0c15256f1f508ec01679f676dd6a4036898a8c2d" +content-hash = "4a0f49b8849dfe6da09b57983368b50b1b40fb96e9d1511108e8ef265c79593c" diff --git a/pyproject.toml b/pyproject.toml index 3966da411..44366c879 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ pgconnstr = "1.0.1" tenacity = "8.2.3" cosl = "0.0.7" pydantic = "1.10.13" +poetry-core = "1.7.0" # psycopg2 = "2.9.5" # Injected in charmcraft.yaml [tool.poetry.group.format] diff --git a/requirements.txt b/requirements.txt index 2d1108444..681a9d0eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ cosl==0.0.7 ; python_full_version >= "3.8.10" and python_full_version < "4.0.0" ops==2.7.0 ; python_full_version >= "3.8.10" and python_full_version < "4.0.0" pgconnstr==1.0.1 ; python_full_version >= "3.8.10" and python_full_version < "4.0.0" +poetry-core==1.7.0 ; python_full_version >= "3.8.10" and python_version < "4.0" pydantic==1.10.13 ; python_full_version >= "3.8.10" and python_full_version < "4.0.0" pyyaml==6.0.1 ; python_full_version >= "3.8.10" and python_full_version < "4.0.0" tenacity==8.2.3 ; python_full_version >= "3.8.10" and python_full_version < "4.0.0" diff --git a/src/charm.py b/src/charm.py index e29a4bdb4..0aa154265 100755 --- a/src/charm.py +++ b/src/charm.py @@ -56,6 +56,7 @@ from relations.db import DbProvides from relations.peers import Peers from relations.pgbouncer_provider import PgBouncerProvider +from upgrade import PgbouncerUpgrade, get_pgbouncer_dependencies_model logger = logging.getLogger(__name__) @@ -100,41 +101,19 @@ def __init__(self, *args): refresh_events=[self.on.config_changed], ) + self.upgrade = PgbouncerUpgrade( + self, + model=get_pgbouncer_dependencies_model(), + relation_name="upgrade", + substrate="vm", + ) + # ======================= # Charm Lifecycle Hooks # ======================= - def _on_install(self, _) -> None: - """On install hook. - - This initialises local config files necessary for pgbouncer to run. - """ - self.unit.status = MaintenanceStatus("Installing and configuring PgBouncer") - - # Install the charmed PostgreSQL snap. - try: - self._install_snap_packages(packages=SNAP_PACKAGES) - except snap.SnapError: - self.unit.status = BlockedStatus("failed to install snap packages") - return - - # Try to disable pgbackrest service - try: - cache = snap.SnapCache() - selected_snap = cache["charmed-postgresql"] - selected_snap.stop(services=["pgbackrest-service"], disable=True) - except snap.SnapError as e: - error_message = "Failed to stop and disable pgbackrest snap service" - logger.exception(error_message, exc_info=e) - - pg_user = pwd.getpwnam(PG_USER) - app_conf_dir = f"{PGB_CONF_DIR}/{self.app.name}" - - # Make a directory for each service to store configs. - for service_id in self.service_ids: - os.makedirs(f"{app_conf_dir}/{INSTANCE_DIR}{service_id}", 0o700, exist_ok=True) - os.chown(f"{app_conf_dir}/{INSTANCE_DIR}{service_id}", pg_user.pw_uid, pg_user.pw_gid) - + def render_utility_files(self): + """Render charm utility services and configuration.""" # Initialise pgbouncer.ini config files from defaults set in charm lib and current config. # We'll add basic configs for now even if this unit isn't a leader, so systemd doesn't # throw a fit. @@ -169,6 +148,39 @@ def _on_install(self, _) -> None: ) ) + def _on_install(self, _) -> None: + """On install hook. + + This initialises local config files necessary for pgbouncer to run. + """ + self.unit.status = MaintenanceStatus("Installing and configuring PgBouncer") + + # Install the charmed PostgreSQL snap. + try: + self._install_snap_packages(packages=SNAP_PACKAGES) + except snap.SnapError: + self.unit.status = BlockedStatus("failed to install snap packages") + return + + # Try to disable pgbackrest service + try: + cache = snap.SnapCache() + selected_snap = cache["charmed-postgresql"] + selected_snap.stop(services=["pgbackrest-service"], disable=True) + except snap.SnapError as e: + error_message = "Failed to stop and disable pgbackrest snap service" + logger.exception(error_message, exc_info=e) + + pg_user = pwd.getpwnam(PG_USER) + app_conf_dir = f"{PGB_CONF_DIR}/{self.app.name}" + + # Make a directory for each service to store configs. + for service_id in self.service_ids: + os.makedirs(f"{app_conf_dir}/{INSTANCE_DIR}{service_id}", 0o700, exist_ok=True) + os.chown(f"{app_conf_dir}/{INSTANCE_DIR}{service_id}", pg_user.pw_uid, pg_user.pw_gid) + + self.render_utility_files() + self.unit.status = WaitingStatus("Waiting to start PgBouncer") def remove_exporter_service(self) -> None: @@ -632,18 +644,20 @@ def render_auth_file(self, auth_file: str, reload_pgbouncer: bool = False): # Charm Utilities # ================= - def _install_snap_packages(self, packages: List[str]) -> None: + def _install_snap_packages(self, packages: List[str], refresh: bool = False) -> None: """Installs package(s) to container. Args: packages: list of packages to install. + refresh: whether to refresh the snap if it's + already present. """ for snap_name, snap_version in packages: try: snap_cache = snap.SnapCache() snap_package = snap_cache[snap_name] - if not snap_package.present: + if not snap_package.present or refresh: if snap_version.get("revision"): snap_package.ensure( snap.SnapState.Latest, revision=snap_version["revision"] diff --git a/src/dependency.json b/src/dependency.json new file mode 100644 index 000000000..275c580ac --- /dev/null +++ b/src/dependency.json @@ -0,0 +1,14 @@ +{ + "charm": { + "dependencies": {}, + "name": "pgbouncer", + "upgrade_supported": ">0", + "version": "1" + }, + "snap": { + "dependencies": {}, + "name": "charmed-postgresql", + "upgrade_supported": "^1", + "version": "1.18.0" + } +} diff --git a/src/relations/peers.py b/src/relations/peers.py index 106afa826..958ddbefa 100644 --- a/src/relations/peers.py +++ b/src/relations/peers.py @@ -52,7 +52,7 @@ """ # noqa: W505 import logging -from typing import Optional, Set +from typing import List, Optional, Set from charms.pgbouncer_k8s.v0.pgb import PgbConfig from ops.charm import CharmBase, RelationChangedEvent, RelationCreatedEvent @@ -131,6 +131,11 @@ def leader_ip(self) -> str: """Gets the IP of the leader unit.""" return self.app_databag.get(LEADER_ADDRESS_KEY, None) + @property + def units(self) -> List[Unit]: + """Returns the peer relation units.""" + return self.relation.units + def _get_unit_ip(self, unit: Unit) -> Optional[str]: """Get the IP address of a specific unit.""" # Check if host is current host. diff --git a/src/upgrade.py b/src/upgrade.py new file mode 100644 index 000000000..0854db286 --- /dev/null +++ b/src/upgrade.py @@ -0,0 +1,111 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Upgrades implementation.""" +import json +import logging +from typing import List + +from charms.data_platform_libs.v0.upgrade import ( + ClusterNotReadyError, + DataUpgrade, + DependencyModel, + UpgradeGrantedEvent, +) +from charms.operator_libs_linux.v1 import systemd +from ops.model import ActiveStatus, MaintenanceStatus +from pydantic import BaseModel +from tenacity import Retrying, stop_after_attempt, wait_fixed +from typing_extensions import override + +from constants import SNAP_PACKAGES + +DEFAULT_MESSAGE = "Pre-upgrade check failed and cannot safely upgrade" + +logger = logging.getLogger(__name__) + + +class PgbouncerDependencyModel(BaseModel): + """Pgbouncer dependencies model.""" + + charm: DependencyModel + snap: DependencyModel + + +def get_pgbouncer_dependencies_model() -> PgbouncerDependencyModel: + """Return the PostgreSQL dependencies model.""" + with open("src/dependency.json") as dependency_file: + _deps = json.load(dependency_file) + return PgbouncerDependencyModel(**_deps) + + +class PgbouncerUpgrade(DataUpgrade): + """PostgreSQL upgrade class.""" + + def __init__(self, charm, model: BaseModel, **kwargs) -> None: + """Initialize the class.""" + super().__init__(charm, model, **kwargs) + self.charm = charm + + @override + def build_upgrade_stack(self) -> List[int]: + """Builds ordered iterable of all application unit.ids to upgrade in.""" + return [ + int(unit.name.split("/")[-1]) + for unit in [self.charm.unit] + list(self.charm.peers.units) + ] + + @override + def log_rollback_instructions(self) -> None: + """Log rollback instructions.""" + logger.info( + "Run `juju refresh --revision pgbouncer` to initiate the rollback" + ) + + def _cluster_checks(self) -> None: + """Check that the cluster is in healthy state.""" + if not isinstance(self.charm.check_status(), ActiveStatus): + raise ClusterNotReadyError(DEFAULT_MESSAGE, "Not all pgbouncer services are up yet.") + + if self.charm.backend.postgres and not self.charm.backend.ready: + raise ClusterNotReadyError(DEFAULT_MESSAGE, "Backend relation is still initialising.") + + @override + def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None: + # Refresh the charmed PostgreSQL snap and restart the database. + self.charm.unit.status = MaintenanceStatus("stopping services") + for service in self.charm.pgb_services: + systemd.service_stop(service) + if self.charm.backend.postgres: + self.charm.remove_exporter_service() + + self.charm.unit.status = MaintenanceStatus("refreshing the snap") + self.charm._install_snap_packages(packages=SNAP_PACKAGES, refresh=True) + + self.charm.unit.status = MaintenanceStatus("restarting services") + self.charm.render_utility_files() + self.charm.reload_pgbouncer() + if self.charm.backend.postgres: + self.charm.render_prometheus_service() + + for attempt in Retrying(stop=stop_after_attempt(6), wait=wait_fixed(10), reraise=True): + with attempt: + self._cluster_checks() + + self.set_unit_completed() + self.charm.unit.status = ActiveStatus() + + # Ensures leader gets its own relation-changed when it upgrades + if self.charm.unit.is_leader(): + self.on_upgrade_changed(event) + + @override + def pre_upgrade_check(self) -> None: + """Runs necessary checks validating the cluster is in a healthy state to upgrade. + + Called by all units during :meth:`_on_pre_upgrade_check_action`. + + Raises: + :class:`ClusterNotReadyError`: if cluster is not ready to upgrade + """ + self._cluster_checks() diff --git a/tests/integration/helpers/helpers.py b/tests/integration/helpers/helpers.py index ce3f31b62..2134fe324 100644 --- a/tests/integration/helpers/helpers.py +++ b/tests/integration/helpers/helpers.py @@ -281,7 +281,7 @@ async def deploy_postgres_bundle( PG, channel="14/edge", num_units=db_units, - config=pg_config, + config={"profile": "testing", **pg_config}, ), ) async with ops_test.fast_forward(): diff --git a/tests/integration/relations/pgbouncer_provider/test_pgbouncer_provider.py b/tests/integration/relations/pgbouncer_provider/test_pgbouncer_provider.py index 1f6361cbd..254f3ba59 100644 --- a/tests/integration/relations/pgbouncer_provider/test_pgbouncer_provider.py +++ b/tests/integration/relations/pgbouncer_provider/test_pgbouncer_provider.py @@ -64,6 +64,7 @@ async def test_database_relation_with_charm_libraries(ops_test: OpsTest, pgb_cha application_name=PG, num_units=2, channel="14/edge", + config={"profile": "testing"}, ), ) await ops_test.model.add_relation(f"{PGB}:{BACKEND_RELATION_NAME}", f"{PG}:database") @@ -238,6 +239,7 @@ async def test_an_application_can_connect_to_multiple_database_clusters( application_name=PG_2, num_units=2, channel="14/edge", + config={"profile": "testing"}, ), ) await ops_test.model.add_relation(f"{PGB_2}:{BACKEND_RELATION_NAME}", f"{PG_2}:database") diff --git a/tests/integration/relations/test_peers.py b/tests/integration/relations/test_peers.py index fa7989e37..4ed432274 100644 --- a/tests/integration/relations/test_peers.py +++ b/tests/integration/relations/test_peers.py @@ -46,7 +46,9 @@ async def test_scaled_relations(ops_test: OpsTest): """Test that the pgbouncer, postgres, and client charms can relate to one another.""" # Build, deploy, and relate charms. async with ops_test.fast_forward(): - await ops_test.model.deploy(PG, channel="14/edge", trust=True, num_units=3), + await ops_test.model.deploy( + PG, channel="14/edge", trust=True, num_units=3, config={"profile": "testing"} + ) await asyncio.gather( ops_test.model.wait_for_idle( diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 95cb7fc19..0df3f7c59 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -44,6 +44,7 @@ async def test_build_and_deploy(ops_test: OpsTest, pgb_charm_jammy): PG, application_name=PG, channel="14/edge", + config={"profile": "testing"}, ), ) # Relate the charms and wait for them exchanging some connection data. diff --git a/tests/integration/test_upgrade.py b/tests/integration/test_upgrade.py new file mode 100644 index 000000000..6ec43c5ce --- /dev/null +++ b/tests/integration/test_upgrade.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +import asyncio +import logging + +import pytest +from pytest_operator.plugin import OpsTest + +from constants import BACKEND_RELATION_NAME +from tests.integration.helpers.helpers import ( + CLIENT_APP_NAME, + FIRST_DATABASE_RELATION_NAME, + PG, + PGB, +) +from tests.integration.relations.pgbouncer_provider.helpers import ( + check_new_relation, +) + +logger = logging.getLogger(__name__) + +DATA_INTEGRATOR_APP_NAME = "data-integrator" +CLIENT_UNIT_NAME = f"{CLIENT_APP_NAME}/0" +TEST_DBNAME = "postgresql_test_app_first_database" +ANOTHER_APPLICATION_APP_NAME = "another-application" +PG_2 = "another-postgresql" +PGB_2 = "another-pgbouncer" +APP_NAMES = [CLIENT_APP_NAME, PG] +MULTIPLE_DATABASE_CLUSTERS_RELATION_NAME = "multiple-database-clusters" + + +@pytest.mark.abort_on_fail +async def test_in_place_upgrade(ops_test: OpsTest, pgb_charm_jammy): + """Test basic functionality of database relation interface.""" + # Deploy both charms (multiple units for each application to test that later they correctly + # set data in the relation application databag using only the leader unit). + logger.info("Deploying PGB...") + async with ops_test.fast_forward(): + await asyncio.gather( + ops_test.model.deploy( + CLIENT_APP_NAME, + application_name=CLIENT_APP_NAME, + channel="edge", + ), + ops_test.model.deploy( + pgb_charm_jammy, + application_name=PGB, + num_units=None, + ), + ops_test.model.deploy( + PG, + application_name=PG, + num_units=2, + channel="14/edge", + config={"profile": "testing"}, + ), + ) + await ops_test.model.add_relation(f"{PGB}:{BACKEND_RELATION_NAME}", f"{PG}:database") + await ops_test.model.wait_for_idle(apps=[PG, CLIENT_APP_NAME], timeout=1200) + # Relate the charms and wait for them exchanging some connection data. + global client_relation + client_relation = await ops_test.model.add_relation( + f"{CLIENT_APP_NAME}:{FIRST_DATABASE_RELATION_NAME}", PGB + ) + + await ops_test.model.wait_for_idle(status="active", timeout=600) + + # This test hasn't passed if we can't pass a tiny amount of data through the new relation + await check_new_relation( + ops_test, + unit_name=ops_test.model.applications[CLIENT_APP_NAME].units[0].name, + relation_id=client_relation.id, + relation_name=FIRST_DATABASE_RELATION_NAME, + dbname=TEST_DBNAME, + ) + + leader = None + for unit in ops_test.model.applications[PGB].units: + if await unit.is_leader_from_status(): + leader = unit + break + + action = await leader.run_action("pre-upgrade-check") + await action.wait() + + await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active", raise_on_blocked=True) + + logger.info("Upgrading PGB...") + await ops_test.model.applications[PGB].refresh(path=pgb_charm_jammy) + await ops_test.model.wait_for_idle(apps=[PGB], status="active", raise_on_blocked=True) + + await check_new_relation( + ops_test, + unit_name=ops_test.model.applications[CLIENT_APP_NAME].units[0].name, + relation_id=client_relation.id, + relation_name=FIRST_DATABASE_RELATION_NAME, + dbname=TEST_DBNAME, + ) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index b11443f55..cbc004d44 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -39,13 +39,12 @@ class TestCharm(unittest.TestCase): def setUp(self): self.harness = Harness(PgBouncerCharm) self.addCleanup(self.harness.cleanup) - with patch("charm.systemd"): - self.harness.begin_with_initial_hooks() + self.harness.begin() self.charm = self.harness.charm self.unit = self.harness.charm.unit - self.rel_id = self.harness.model.relations[PEER_RELATION_NAME][0].id + self.rel_id = self.harness.add_relation(PEER_RELATION_NAME, self.charm.app.name) @patch("builtins.open", unittest.mock.mock_open()) @patch("charm.PgBouncerCharm._install_snap_packages") diff --git a/tests/unit/test_upgrade.py b/tests/unit/test_upgrade.py new file mode 100644 index 000000000..746caec69 --- /dev/null +++ b/tests/unit/test_upgrade.py @@ -0,0 +1,138 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +import unittest +from unittest.mock import Mock, PropertyMock, patch + +import pytest +import tenacity +from charms.data_platform_libs.v0.upgrade import ClusterNotReadyError +from ops.model import ActiveStatus, MaintenanceStatus +from ops.testing import Harness + +from charm import PgBouncerCharm +from constants import SNAP_PACKAGES + + +class TestUpgrade(unittest.TestCase): + def setUp(self): + self.harness = Harness(PgBouncerCharm) + self.addCleanup(self.harness.cleanup) + self.harness.begin() + + self.charm = self.harness.charm + self.unit = self.harness.charm.unit + + @patch("charm.Peers.units", new_callable=PropertyMock) + def test_build_upgrade_stack(self, _peers: Mock): + _peers.return_value = [Mock(), Mock()] + _peers.return_value[0].name = "test/1" + _peers.return_value[1].name = "test/2" + + result = self.charm.upgrade.build_upgrade_stack() + + assert result == [0, 1, 2] + + @patch("upgrade.logger.info") + def test_log_rollback_instructions(self, _logger: Mock): + self.charm.upgrade.log_rollback_instructions() + + _logger.assert_called_once_with( + "Run `juju refresh --revision pgbouncer` to initiate the rollback" + ) + + @patch("charm.BackendDatabaseRequires.postgres", return_value=True, new_callable=PropertyMock) + @patch("charm.PgbouncerUpgrade.on_upgrade_changed") + @patch("charm.PgbouncerUpgrade.set_unit_completed") + @patch("charm.PgbouncerUpgrade._cluster_checks") + @patch("charm.PgBouncerCharm.render_prometheus_service") + @patch("charm.PgBouncerCharm.reload_pgbouncer") + @patch("charm.PgBouncerCharm.render_utility_files") + @patch("charm.PgBouncerCharm._install_snap_packages") + @patch("charm.PgBouncerCharm.remove_exporter_service") + @patch("upgrade.systemd") + def test_on_upgrade_granted( + self, + _systemd: Mock, + _remove_exporter_service: Mock, + _install_snap_packages: Mock, + _render_utility_files: Mock, + _reload_pgbouncer: Mock, + _render_prometheus_service: Mock, + _cluster_checks: Mock, + _set_unit_completed: Mock, + _on_upgrade_changed: Mock, + _, + ): + event = Mock() + + self.charm.upgrade._on_upgrade_granted(event) + + assert _systemd.service_stop.call_count == len(self.charm.pgb_services) + for svc in self.charm.pgb_services: + _systemd.service_stop.assert_any_call(svc) + _remove_exporter_service.assert_called_once_with() + _install_snap_packages.assert_called_once_with(packages=SNAP_PACKAGES, refresh=True) + _render_prometheus_service.assert_called_once_with() + _cluster_checks.assert_called_once_with() + _set_unit_completed.assert_called_once_with() + + # Test extra call as leader + with self.harness.hooks_disabled(): + self.harness.set_leader(True) + + self.charm.upgrade._on_upgrade_granted(event) + + _on_upgrade_changed.assert_called_once_with(event) + + @patch("upgrade.wait_fixed", return_value=tenacity.wait_fixed(0)) + @patch("charm.BackendDatabaseRequires.postgres", return_value=True, new_callable=PropertyMock) + @patch("charm.PgbouncerUpgrade.on_upgrade_changed") + @patch("charm.PgbouncerUpgrade.set_unit_completed") + @patch("charm.PgbouncerUpgrade._cluster_checks") + @patch("charm.PgBouncerCharm.render_prometheus_service") + @patch("charm.PgBouncerCharm.reload_pgbouncer") + @patch("charm.PgBouncerCharm.render_utility_files") + @patch("charm.PgBouncerCharm._install_snap_packages") + @patch("charm.PgBouncerCharm.remove_exporter_service") + @patch("upgrade.systemd") + def test_on_upgrade_granted_error( + self, + _systemd: Mock, + _remove_exporter_service: Mock, + _install_snap_packages: Mock, + _render_utility_files: Mock, + _reload_pgbouncer: Mock, + _render_prometheus_service: Mock, + _cluster_checks: Mock, + _set_unit_completed: Mock, + _on_upgrade_changed: Mock, + _, + __, + ): + _cluster_checks.side_effect = ClusterNotReadyError("test", "test") + + with pytest.raises(ClusterNotReadyError): + self.charm.upgrade._on_upgrade_granted(Mock()) + + @patch("charm.PgBouncerCharm.check_status", return_value=ActiveStatus()) + def test_pre_upgrade_check(self, _check_status: Mock): + self.charm.upgrade.pre_upgrade_check() + + _check_status.assert_called_once_with() + + @patch("charm.PgBouncerCharm.check_status", return_value=MaintenanceStatus()) + def test_pre_upgrade_check_not_ready(self, _check_status: Mock): + with pytest.raises(ClusterNotReadyError): + self.charm.upgrade.pre_upgrade_check() + + _check_status.assert_called_once_with() + + @patch("charm.BackendDatabaseRequires.postgres", return_value=True, new_callable=PropertyMock) + @patch("charm.BackendDatabaseRequires.ready", return_value=False, new_callable=PropertyMock) + @patch("charm.PgBouncerCharm.check_status", return_value=ActiveStatus()) + def test_pre_upgrade_check_backend_not_ready(self, _check_status: Mock, _, __): + print(self.charm.backend.ready) + with pytest.raises(ClusterNotReadyError): + self.charm.upgrade.pre_upgrade_check() + + _check_status.assert_called_once_with() diff --git a/tox.ini b/tox.ini index 38b4cbe55..fd6711741 100644 --- a/tox.ini +++ b/tox.ini @@ -130,6 +130,17 @@ commands = pip install juju=={env:LIBJUJU} poetry run pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/relations/test_peers.py +[testenv:upgrade-integration-{juju2, juju3}] +description = Run integration tests for upgrade +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS +commands = + poetry install --with integration + pip install juju=={env:LIBJUJU} + poetry run pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/test_upgrade.py + [testenv:poetry-lock] description = Install, lock and export poetry dependencies commands =