From 1b4f63f4e47960b4afee5abd334543c6d79cf5b6 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Fri, 30 Aug 2024 17:28:08 +0200 Subject: [PATCH 01/40] WIP mongodb integration --- .github/workflows/integration_test.yaml | 1 + examples/flask/charmcraft.yaml | 5 + paas_app_charmer/app.py | 11 ++ paas_app_charmer/charm.py | 46 ++++- paas_app_charmer/charm_state.py | 32 +++- paas_app_charmer/rabbitmq.py | 181 +++++++++++++++++++ tests/integration/flask/test_integrations.py | 26 +++ tests/unit/flask/test_charm.py | 67 ++++++- 8 files changed, 365 insertions(+), 4 deletions(-) create mode 100644 paas_app_charmer/rabbitmq.py diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 0751bab..5f760f1 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -19,3 +19,4 @@ jobs: rockcraft-ref: fastapi-extension juju-channel: ${{ matrix.juju-version }} channel: 1.29-strict/stable + tmate-debug: true diff --git a/examples/flask/charmcraft.yaml b/examples/flask/charmcraft.yaml index 160f2d7..ae92bf5 100644 --- a/examples/flask/charmcraft.yaml +++ b/examples/flask/charmcraft.yaml @@ -115,6 +115,11 @@ requires: interface: saml optional: True limit: 1 + amqp: + interface: rabbitmq + optional: True + limit: 1 + resources: flask-app-image: description: flask application image. diff --git a/paas_app_charmer/app.py b/paas_app_charmer/app.py index 678e5db..41010ba 100644 --- a/paas_app_charmer/app.py +++ b/paas_app_charmer/app.py @@ -259,6 +259,17 @@ def map_integrations_to_env(integrations: IntegrationsState, prefix: str = "") - if v is not None ) + if integrations.rabbitmq_parameters: + rabbitmq = integrations.rabbitmq_parameters + env.update( + ( + ("RABBITMQ_HOSTNAME", rabbitmq.hostname), + ("RABBITMQ_USERNAME", rabbitmq.username), + ("RABBITMQ_PASSWORD", rabbitmq.password), + ("RABBITMQ_VHOST", rabbitmq.vhost), + ) + ) + return {prefix + k: v for k, v in env.items()} diff --git a/paas_app_charmer/charm.py b/paas_app_charmer/charm.py index 6f85720..37ee9a4 100644 --- a/paas_app_charmer/charm.py +++ b/paas_app_charmer/charm.py @@ -19,6 +19,7 @@ from paas_app_charmer.databases import make_database_requirers from paas_app_charmer.exceptions import CharmConfigInvalidError from paas_app_charmer.observability import Observability +from paas_app_charmer.rabbitmq import RabbitMQRequires from paas_app_charmer.secret_storage import KeySecretStorage from paas_app_charmer.utils import build_validation_error_message @@ -101,6 +102,26 @@ def __init__(self, framework: ops.Framework, framework_name: str) -> None: else: self._saml = None + self._amqp: RabbitMQRequires | None + if "amqp" in requires and requires["amqp"].interface_name == "rabbitmq": + logger.info("JAVI PREPARING RABBIT REQUIRES") + self._amqp = RabbitMQRequires( + self, + "amqp", + username=self.app.name, + vhost="/", + ) + logger.info("JAVI rabbitmq_requires._amqp_rel %s", str(self._amqp._amqp_rel)) + if self._amqp._amqp_rel is not None: + logger.info( + "JAVI rabbitmq_requires._amqp_rel.data %s", str(self._amqp._amqp_rel.data) + ) + self.framework.observe(self._amqp.on.connected, self._on_amqp_connected) + self.framework.observe(self._amqp.on.ready, self._on_amqp_ready) + self.framework.observe(self._amqp.on.goneaway, self._on_amqp_goneaway) + else: + self._amqp = None + self._database_migration = DatabaseMigration( container=self.unit.get_container(self._workload_config.container_name), state_dir=self._workload_config.state_dir, @@ -245,7 +266,8 @@ def is_ready(self) -> bool: return True - def _missing_required_integrations(self, charm_state: CharmState) -> list[str]: + # JAVI PENDING TO REFACTOR + def _missing_required_integrations(self, charm_state: CharmState) -> list[str]: # noqa: C901 """Get list of missing integrations that are required. Args: @@ -272,6 +294,9 @@ def _missing_required_integrations(self, charm_state: CharmState) -> list[str]: if self._saml and not charm_state.integrations.saml_parameters: if not requires["saml"].optional: missing_integrations.append("saml") + if self._amqp and not charm_state.integrations.rabbitmq_parameters: + if not requires["amqp"].optional: + missing_integrations.append("amqp") return missing_integrations def restart(self) -> None: @@ -319,6 +344,7 @@ def _create_charm_state(self) -> CharmState: redis_uri=self._redis.url if self._redis is not None else None, s3_connection_info=self._s3.get_s3_connection_info() if self._s3 else None, saml_relation_data=saml_relation_data, + rabbitmq_parameters=self._amqp.rabbitmq_parameters() if self._amqp else None, base_url=self._base_url, ) @@ -418,3 +444,21 @@ def _on_ingress_ready(self, _: ops.HookEvent) -> None: def _on_pebble_ready(self, _: ops.PebbleReadyEvent) -> None: """Handle the pebble-ready event.""" self.restart() + + @block_if_invalid_config + def _on_amqp_connected(self, _event: ops.HookEvent) -> None: + """Handle ampq connected event.""" + logger.info("_ON_AMQP_CONNECTED") + self.restart() + + @block_if_invalid_config + def _on_amqp_ready(self, _event: ops.HookEvent) -> None: + """Handle ampq ready event.""" + logger.info("_ON_AMQP_READY") + self.restart() + + @block_if_invalid_config + def _on_amqp_goneaway(self, _event: ops.HookEvent) -> None: + """Handle ampq goneaway event.""" + logger.info("_ON_AMQP_GONEAWAY") + self.restart() diff --git a/paas_app_charmer/charm_state.py b/paas_app_charmer/charm_state.py index ac920e8..47723f8 100644 --- a/paas_app_charmer/charm_state.py +++ b/paas_app_charmer/charm_state.py @@ -88,6 +88,7 @@ def from_charm( # pylint: disable=too-many-arguments redis_uri: str | None = None, s3_connection_info: dict[str, str] | None = None, saml_relation_data: typing.MutableMapping[str, str] | None = None, + rabbitmq_parameters: "RabbitMQParameters | None" = None, base_url: str | None = None, ) -> "CharmState": """Initialize a new instance of the CharmState class from the associated charm. @@ -101,6 +102,7 @@ def from_charm( # pylint: disable=too-many-arguments redis_uri: The redis uri provided by the redis charm. s3_connection_info: Connection info from S3 lib. saml_relation_data: Relation data from the SAML app. + rabbitmq_parameters: JAVI TODO. base_url: Base URL for the service. Return: @@ -120,6 +122,7 @@ def from_charm( # pylint: disable=too-many-arguments database_requirers=database_requirers, s3_connection_info=s3_connection_info, saml_relation_data=saml_relation_data, + rabbitmq_parameters=rabbitmq_parameters, ) return cls( framework=framework, @@ -205,20 +208,23 @@ class IntegrationsState: databases_uris: Map from interface_name to the database uri. s3_parameters: S3 parameters. saml_parameters: SAML parameters. + rabbitmq_parameters: RabbitMQ parameters. """ redis_uri: str | None = None databases_uris: dict[str, str] = field(default_factory=dict) s3_parameters: "S3Parameters | None" = None saml_parameters: "SamlParameters | None" = None + rabbitmq_parameters: "RabbitMQParameters | None" = None @classmethod - def build( + def build( # pylint: disable=too-many-arguments cls, redis_uri: str | None, database_requirers: dict[str, DatabaseRequires], s3_connection_info: dict[str, str] | None, saml_relation_data: typing.MutableMapping[str, str] | None = None, + rabbitmq_parameters: "RabbitMQParameters | None" = None, ) -> "IntegrationsState": """Initialize a new instance of the IntegrationsState class. @@ -229,6 +235,7 @@ def build( database_requirers: All database requirers object declared by the charm. s3_connection_info: S3 connection info from S3 lib. saml_relation_data: Saml relation data from saml lib. + rabbitmq_parameters: TODO JAVI Return: The IntegrationsState instance created. @@ -276,6 +283,7 @@ def build( }, s3_parameters=s3_parameters, saml_parameters=saml_parameters, + rabbitmq_parameters=rabbitmq_parameters, ) @@ -354,3 +362,25 @@ def validate_signing_certificate_exists(cls, certs: str, _: ValidationInfo) -> s if not certificate: raise ValueError("Missing x509certs. There should be at least one certificate.") return certificate + + +class RabbitMQParameters(BaseModel, extra=Extra.allow): + """Configuration for accessing RabbitMQ. + + Attributes: + hostname: Hostname or ip address of broker. It is one of the hostnames. + hostnames: List of Hostnames or ip addresses of broker. + username: The username to authenticate with + password: The password to authenticate with + vhost: RabbitMQ virtual host name. + """ + + # In rabbitmq-k8s, it is the service url + # In rabbitmq-server, paas-app-charmer puts one of the ips of hosnames arbitrarily + hostname: str + # In rabbitmq-k8s, a list of k8s ingress-address (maybe the same one) + # In rabbitmq-server, the ips of the machines + hostnames: list[str] + username: str + password: str + vhost: str diff --git a/paas_app_charmer/rabbitmq.py b/paas_app_charmer/rabbitmq.py new file mode 100644 index 0000000..5ae5bc9 --- /dev/null +++ b/paas_app_charmer/rabbitmq.py @@ -0,0 +1,181 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""RabbitMQ library for handling the rabbitmq interface. + +The project https://github.com/openstack-charmers/charm-rabbitmq-k8s provides +a library for the requires part of the rabbitmq interface. + +However, there are two charms that provide the rabbitmq interface, being incompatible: + - https://github.com/openstack-charmers/charm-rabbitmq-ks8 (https://charmhub.io/rabbitmq-k8s) + - https://github.com/openstack/charm-rabbitmq-server/ (https://charmhub.io/rabbitmq-server) + +The main difference is that rabbitmq-server does not publish the information in the app +part in the relation bag. This python library unifies both charms, using a similar +approach to the rabbitmq-k8s library. + +# rabbitmq-k8s +# unit-flask-k8s-0: 15:27:43 INFO unit.flask-k8s/0.juju-log JAVI rabbitmq_requires._amqp_rel.data {: {'egress-subnets': '10.152.183.168/32', 'ingress-address': '10.152.183.168', 'private-address': '10.152.183.168'}, : {'username': 'flask-k8s', 'vhost': '/'}, : {'egress-subnets': '10.12.97.225/32', 'ingress-address': '10.12.97.225', 'private-address': '10.12.97.225'}, : {'egress-subnets': '10.12.97.225/32', 'ingress-address': '10.12.97.225', 'private-address': '10.12.97.225'}, : {'hostname': 'rabbitmq-k8s-endpoints.testing.svc.cluster.local', 'password': '3m036hhyiDHs'}} # noqa: W505 # pylint: disable=line-too-long + +# rabbitmq-server +# unit-flask-k8s-0: 16:00:16 INFO unit.flask-k8s/0.juju-log amqp:3: JAVI rabbitmq_requires._amqp_rel.data {: {'egress-subnets': '10.152.183.168/32', 'ingress-address': '10.152.183.168', 'private-address': '10.152.183.168'}, : {'username': 'flask-k8s', 'vhost': '/'}, : {'hostname': '10.58.171.158', 'password': 'LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8', 'private-address': '10.58.171.158'}, : {'hostname': '10.58.171.70', 'password': 'LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8', 'private-address': '10.58.171.70'}, : {'egress-subnets': '10.58.171.146/32', 'hostname': '10.58.171.146', 'ingress-address': '10.58.171.146', 'password': 'LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8', 'private-address': '10.58.171.146'}, : {}} +""" + + +# JAVI COPYPASTED FOR NOW! +# JAVI EXPLAIN NO SSL CLIENT CERFICATES YET +import logging +from typing import cast + +from ops import CharmBase, HookEvent +from ops.framework import EventBase, EventSource, Object, ObjectEvents +from ops.model import Relation +from pydantic import ValidationError + +from paas_app_charmer.charm_state import RabbitMQParameters +from paas_app_charmer.utils import build_validation_error_message + +logger = logging.getLogger(__name__) + + +class RabbitMQConnectedEvent(EventBase): + """RabbitMQ connected Event.""" + + +class RabbitMQReadyEvent(EventBase): + """RabbitMQ ready for use Event.""" + + +class RabbitMQGoneAwayEvent(EventBase): + """RabbitMQ relation has gone-away Event.""" + + +class RabbitMQServerEvents(ObjectEvents): + """Events class for `on`. + + Attributes: + connected: JAVI TODO + ready: JAVI TODO + goneaway: JAVI TODO + """ + + connected = EventSource(RabbitMQConnectedEvent) + ready = EventSource(RabbitMQReadyEvent) + goneaway = EventSource(RabbitMQGoneAwayEvent) + + +class RabbitMQRequires(Object): + """RabbitMQRequires class. + + Attributes: + on: JAVI TODO + """ + + on = RabbitMQServerEvents() + + def __init__(self, charm: CharmBase, relation_name: str, username: str, vhost: str): + """JAVI TODO. + + Args: + charm: JAVI TODO + relation_name: JAVI TODO + username: JAVI TODO + vhost: JAVI TODO + """ + super().__init__(charm, relation_name) + self.charm = charm + self.relation_name = relation_name + self.username = username + self.vhost = vhost + self.framework.observe( + self.charm.on[relation_name].relation_joined, + self._on_amqp_relation_joined, + ) + self.framework.observe( + self.charm.on[relation_name].relation_changed, + self._on_amqp_relation_changed, + ) + self.framework.observe( + self.charm.on[relation_name].relation_departed, + self._on_amqp_relation_changed, + ) + self.framework.observe( + self.charm.on[relation_name].relation_broken, + self._on_amqp_relation_broken, + ) + + def _on_amqp_relation_joined(self, _: HookEvent) -> None: + """JAVI TODO.""" + logging.debug("RabbitMQRabbitMQRequires on_joined") + self.on.connected.emit() + self.request_access(self.username, self.vhost) + + def _on_amqp_relation_changed(self, _: HookEvent) -> None: + """JAVI TODO.""" + logging.debug("RabbitMQRabbitMQRequires on_changed/departed") + if self.rabbitmq_parameters(): + self.on.ready.emit() + + def _on_amqp_relation_broken(self, _: HookEvent) -> None: + """JAVI TODO.""" + logging.debug("RabbitMQRabbitMQRequires on_broken") + self.on.goneaway.emit() + + @property + def _amqp_rel(self) -> Relation | None: + """The RabbitMQ relation.""" + return self.framework.model.get_relation(self.relation_name) + + def rabbitmq_parameters(self) -> RabbitMQParameters | None: + """TODO JAVI. + + Returns: + TODO JAVI + """ + if self._amqp_rel: + # just put everything. + hostnames = [] + password = None + for unit in self._amqp_rel.units: + unit_data = self._amqp_rel.data[unit] + ingress_address = unit_data.get("ingress-address") + unit_hostname = unit_data.get("hostname", ingress_address) + if unit_hostname: + hostnames.append(unit_hostname) + # all of them should be equal + password = unit_data.get("password", password) + + password = self._amqp_rel.data[self._amqp_rel.app].get("password", password) + first_hostname = hostnames[0] if len(hostnames) > 0 else None + hostname = self._amqp_rel.data[self._amqp_rel.app].get("hostname", first_hostname) + + try: + return RabbitMQParameters( + hostname=cast(str, hostname), + hostnames=hostnames, + username=self.username, + password=cast(str, password), + vhost=self.vhost, + ) + except ValidationError as exc: + # do not crash, as the relation can be starting at this moment, and the + # data not being there in the current hook + error_message = build_validation_error_message(exc) + logger.info("Error validating RabbitMQ parameters %s", error_message) + return None + return None + + def request_access(self, username: str, vhost: str) -> None: + """Request access to the RabbitMQ server. + + Args: + username: JAVI TODO + vhost: JAVI TODO + """ + if self.model.unit.is_leader(): + logging.debug("Requesting RabbitMQ user and vhost") + if self._amqp_rel: + self._amqp_rel.data[self.charm.app]["username"] = username + self._amqp_rel.data[self.charm.app]["vhost"] = vhost + else: + logger.warning("request_access but no rabbitmq relation") diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index ad0199e..2ca9c43 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -127,3 +127,29 @@ async def test_saml_integration( entity_id_url._replace(path="sso") ) assert env["SAML_SIGNING_CERTIFICATE"] in saml_helper.CERTIFICATE.replace("\n", "") + + +async def test_rabbitmq_k8s_integration( + ops_test: OpsTest, + flask_app: Application, + model: Model, + get_unit_ips, +): + """ + arrange: TODO + act: TODO + assert: TODO + """ + assert False + + +# juju deploy rabbitmq-k8s --trust --channel=3.12/edge +# juju integrate flask-k8s rabbitmq-k8s + +# lxd: +# juju switch lxd +# juju add-model whatever +# juju deploy rabbitmq-server +# juju offer rabbitmq-server:amqp +# juju switch microk8s +# juju integrate lxd:admin/welcome-lxd.rabbitmq-server flask-k8s diff --git a/tests/unit/flask/test_charm.py b/tests/unit/flask/test_charm.py index a1cba65..6feea23 100644 --- a/tests/unit/flask/test_charm.py +++ b/tests/unit/flask/test_charm.py @@ -123,7 +123,7 @@ def test_ingress(harness: Harness): def test_integrations_wiring(harness: Harness): """ - arrange: Prepare a Redis a database and a S3 integration + arrange: Prepare a Redis a database, a S3 integration and a SAML integration act: Start the flask charm and set flask-app container to be ready. assert: The flask service should have environment variables in its plan for each of the integrations. @@ -164,12 +164,75 @@ def test_integrations_wiring(harness: Harness): assert service_env["SAML_ENTITY_ID"] == SAML_APP_RELATION_DATA_EXAMPLE["entity_id"] +@pytest.mark.parametrize( + "amqp_relation_data,expected_env_vars", + [ + pytest.param( + { + "app_data": { + "hostname": "rabbitmq-k8s-endpoints.testing.svc.cluster.local", + "password": "3m036hhyiDHs", + }, + "unit_data": { + "egress-subnets": "10.152.183.168/32", + "ingress-address": "10.152.183.168", + "private-address": "10.152.183.168", + }, + }, + { + "RABBITMQ_HOSTNAME": "rabbitmq-k8s-endpoints.testing.svc.cluster.local", + "RABBITMQ_USERNAME": "flask-k8s", + "RABBITMQ_PASSWORD": "3m036hhyiDHs", + "RABBITMQ_VHOST": "/", + }, + id="rabbitmq-k8s version", + ), + pytest.param( + { + "app_data": {}, + "unit_data": { + "hostname": "10.58.171.158", + "password": "LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8", + "private-address": "10.58.171.158", + }, + }, + { + "RABBITMQ_HOSTNAME": "10.58.171.158", + "RABBITMQ_USERNAME": "flask-k8s", + "RABBITMQ_PASSWORD": "LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8", + "RABBITMQ_VHOST": "/", + }, + id="rabbitmq-server version", + ), + ], +) +def test_rabbitmq_integration(harness: Harness, amqp_relation_data, expected_env_vars): + """ + arrange: Prepare a amqp integration (RabbitMQ) + act: Start the flask charm and set flask-app container to be ready. + assert: The flask service should have environment variables in its plan + for each of the integrations. + """ + + harness.add_relation("amqp", "amqp", **amqp_relation_data) + container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) + container.add_layer("a_layer", DEFAULT_LAYER) + harness.begin_with_initial_hooks() + assert harness.model.unit.status == ops.ActiveStatus() + service_env = container.get_plan().services["flask"].environment + for env, env_val in expected_env_vars.items(): + assert env in service_env + assert service_env[env] == env_val + + @pytest.mark.parametrize( "integrate_to,required_integrations", [ pytest.param(["saml"], ["s3"], id="s3 fails"), pytest.param(["redis", "s3"], ["mysql", "postgresql"], id="postgresql and mysql fail"), - pytest.param([], ["mysql", "postgresql", "mongodb", "s3", "redis", "saml"], id="all fail"), + pytest.param( + [], ["mysql", "postgresql", "mongodb", "s3", "redis", "saml", "amqp"], id="all fail" + ), ], ) def test_missing_integrations(harness: Harness, integrate_to, required_integrations): From 83a5e9054600cbab05af68db9d4337c5f11acb45 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Mon, 2 Sep 2024 14:53:28 +0200 Subject: [PATCH 02/40] fail a bit faster --- tests/integration/flask/test_integrations.py | 51 ++++++++++---------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index 2ca9c43..5c95e4c 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -16,6 +16,32 @@ logger = logging.getLogger(__name__) +async def test_rabbitmq_k8s_integration( + ops_test: OpsTest, + flask_app: Application, + model: Model, + get_unit_ips, +): + """ + arrange: TODO + act: TODO + assert: TODO + """ + assert False + + +# juju deploy rabbitmq-k8s --trust --channel=3.12/edge +# juju integrate flask-k8s rabbitmq-k8s + +# lxd: +# juju switch lxd +# juju add-model whatever +# juju deploy rabbitmq-server +# juju offer rabbitmq-server:amqp +# juju switch microk8s +# juju integrate lxd:admin/welcome-lxd.rabbitmq-server flask-k8s + + async def test_s3_integration( ops_test: OpsTest, flask_app: Application, @@ -128,28 +154,3 @@ async def test_saml_integration( ) assert env["SAML_SIGNING_CERTIFICATE"] in saml_helper.CERTIFICATE.replace("\n", "") - -async def test_rabbitmq_k8s_integration( - ops_test: OpsTest, - flask_app: Application, - model: Model, - get_unit_ips, -): - """ - arrange: TODO - act: TODO - assert: TODO - """ - assert False - - -# juju deploy rabbitmq-k8s --trust --channel=3.12/edge -# juju integrate flask-k8s rabbitmq-k8s - -# lxd: -# juju switch lxd -# juju add-model whatever -# juju deploy rabbitmq-server -# juju offer rabbitmq-server:amqp -# juju switch microk8s -# juju integrate lxd:admin/welcome-lxd.rabbitmq-server flask-k8s From 0edbb736e36c6989ca3c508255abbaa8555272b1 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Mon, 2 Sep 2024 18:09:45 +0200 Subject: [PATCH 03/40] Let's see if it works --- tests/integration/conftest.py | 46 +++++++++++++++++++- tests/integration/flask/conftest.py | 14 ++++++ tests/integration/flask/test_integrations.py | 17 ++++++-- 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0a6a41d..f8d08bd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,12 +2,56 @@ # See LICENSE file for licensing details. import json +import logging import pytest import pytest_asyncio -from juju.model import Model +from juju.application import Application +from juju.juju import Juju +from juju.model import Controller, Model from pytest_operator.plugin import OpsTest +logger = logging.getLogger(__name__) + + +@pytest_asyncio.fixture(scope="module", name="lxd_controller") +async def fixture_lxd_controller(ops_test: OpsTest) -> Controller: + """Return the current testing juju model.""" + if not "lxd" in Juju().get_controllers(): + logger.info("creating lxd controller") + _, _, _ = await ops_test.juju("bootstrap", "localhost", "lxd", check=True) + + controller = Controller() + logger.info("connecting to lxd controller") + await controller.connect("lxd") + yield controller + logger.info("disconnecting from lxd controller") + await controller.disconnect() + + +@pytest_asyncio.fixture(scope="module", name="lxd_model") +async def fixture_lxd_model(lxd_controller: Controller) -> Model: + model = await lxd_controller.add_model("lxd") + yield model + logger.info("destroy model %s", model.name) + # await lxd_controller.destroy_models(model.name, destroy_storage=True, force=True, max_wait=60) + + +@pytest_asyncio.fixture(scope="module", name="rabbitmq_server_app") # autouse=True) +async def deploy_rabbitmq_server_fixture( + lxd_model: Model, +) -> Application: + """Deploy rabbitmq-server machine app.""" + app = await lxd_model.deploy( + "rabbitmq-server", + # channel="3.9/stable", + channel="latest/edge", + ) + await lxd_model.wait_for_idle(raise_on_blocked=True) + offer = await lxd_model.create_offer("rabbitmq-server:amqp") + logger.info("offer: %s", offer) + yield app + @pytest_asyncio.fixture(scope="module", name="get_unit_ips") async def fixture_get_unit_ips(ops_test: OpsTest): diff --git a/tests/integration/flask/conftest.py b/tests/integration/flask/conftest.py index 0aa0482..9865318 100644 --- a/tests/integration/flask/conftest.py +++ b/tests/integration/flask/conftest.py @@ -21,6 +21,20 @@ PROJECT_ROOT = pathlib.Path(__file__).parent.parent.parent.parent +import asyncio + +import nest_asyncio + + +@pytest.fixture(scope="module") +def event_loop(): + nest_asyncio.apply() + loop = asyncio.new_event_loop() + asyncio._set_running_loop(loop) + yield loop + loop.close() + + @pytest.fixture(autouse=True) def cwd(): return os.chdir(PROJECT_ROOT / "examples/flask") diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index 5c95e4c..62e5e89 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -16,10 +16,12 @@ logger = logging.getLogger(__name__) -async def test_rabbitmq_k8s_integration( +async def test_rabbitmq_server_integration( ops_test: OpsTest, flask_app: Application, + rabbitmq_server_app: Application, model: Model, + lxd_model: Model, get_unit_ips, ): """ @@ -27,12 +29,22 @@ async def test_rabbitmq_k8s_integration( act: TODO assert: TODO """ - assert False + + # TODO GET THE OOFFER NAME FROM THE OFFER CREATION? + # integration = await model.integrate("lxd:admin/lxd.rabbitmq-server", flask_app.name) + + integration = await model.integrate("rabbitmq-server", flask_app.name) + + lxd_status = await lxd_model.get_status() + # get ip from lxd_status + + assert True # juju deploy rabbitmq-k8s --trust --channel=3.12/edge # juju integrate flask-k8s rabbitmq-k8s +# juju bootstrap localhost lxd # lxd: # juju switch lxd # juju add-model whatever @@ -153,4 +165,3 @@ async def test_saml_integration( entity_id_url._replace(path="sso") ) assert env["SAML_SIGNING_CERTIFICATE"] in saml_helper.CERTIFICATE.replace("\n", "") - From ff5ea46a4934772e633f46a666320445cfe6a020 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Mon, 2 Sep 2024 21:42:58 +0200 Subject: [PATCH 04/40] comment nestasyncio --- tests/integration/flask/conftest.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/integration/flask/conftest.py b/tests/integration/flask/conftest.py index 9865318..7eca1eb 100644 --- a/tests/integration/flask/conftest.py +++ b/tests/integration/flask/conftest.py @@ -23,16 +23,16 @@ import asyncio -import nest_asyncio +# import nest_asyncio -@pytest.fixture(scope="module") -def event_loop(): - nest_asyncio.apply() - loop = asyncio.new_event_loop() - asyncio._set_running_loop(loop) - yield loop - loop.close() +# @pytest.fixture(scope="module") +# def event_loop(): +# nest_asyncio.apply() +# loop = asyncio.new_event_loop() +# asyncio._set_running_loop(loop) +# yield loop +# loop.close() @pytest.fixture(autouse=True) From 699800d334626d366d5238b3ce6012a8d72bb15d Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Mon, 2 Sep 2024 22:16:53 +0200 Subject: [PATCH 05/40] integrate with full name --- tests/integration/flask/test_integrations.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index 62e5e89..91233c1 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -31,9 +31,8 @@ async def test_rabbitmq_server_integration( """ # TODO GET THE OOFFER NAME FROM THE OFFER CREATION? - # integration = await model.integrate("lxd:admin/lxd.rabbitmq-server", flask_app.name) - - integration = await model.integrate("rabbitmq-server", flask_app.name) + integration = await model.integrate("lxd:lxd.rabbitmq-server", flask_app.name) + # integration = await model.integrate("rabbitmq-server", flask_app.name) lxd_status = await lxd_model.get_status() # get ip from lxd_status From 29b6e1067b72a7f7d4fe78cf1c69ad0a3ffcb2fe Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Mon, 2 Sep 2024 23:32:34 +0200 Subject: [PATCH 06/40] put name --- tests/integration/flask/test_integrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index 91233c1..304b4b9 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -31,7 +31,7 @@ async def test_rabbitmq_server_integration( """ # TODO GET THE OOFFER NAME FROM THE OFFER CREATION? - integration = await model.integrate("lxd:lxd.rabbitmq-server", flask_app.name) + integration = await model.integrate("lxd:admin/lxd.rabbitmq-server", flask_app.name) # integration = await model.integrate("rabbitmq-server", flask_app.name) lxd_status = await lxd_model.get_status() From e110ddec548af25edc8ea612a3644dd4d56a56b7 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 3 Sep 2024 09:59:45 +0200 Subject: [PATCH 07/40] switch cli client to its original controller --- tests/integration/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f8d08bd..34001dc 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,6 +7,7 @@ import pytest import pytest_asyncio from juju.application import Application +from juju.client.jujudata import FileJujuData from juju.juju import Juju from juju.model import Controller, Model from pytest_operator.plugin import OpsTest @@ -18,8 +19,13 @@ async def fixture_lxd_controller(ops_test: OpsTest) -> Controller: """Return the current testing juju model.""" if not "lxd" in Juju().get_controllers(): - logger.info("creating lxd controller") + jujudata = FileJujuData() + previous_controller = jujudata.current_controller() + logger.info("bootstrapping lxd") _, _, _ = await ops_test.juju("bootstrap", "localhost", "lxd", check=True) + # go back to the original controller + logger.info("switch back to %s", previous_controller) + _, _, _ = await ops_test.juju("switch", previous_controller, check=True) controller = Controller() logger.info("connecting to lxd controller") From 0e5b3849a06f1f2a3f06cea5049a502ad32a5f75 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 3 Sep 2024 14:28:06 +0200 Subject: [PATCH 08/40] try lxd + not lxd --- examples/flask/test_rock/app.py | 42 +++++++++++++ examples/flask/test_rock/requirements.txt | 1 + tests/integration/conftest.py | 45 +++++++++----- tests/integration/flask/conftest.py | 11 ---- tests/integration/flask/test_integrations.py | 64 +++++++++++++++----- 5 files changed, 121 insertions(+), 42 deletions(-) diff --git a/examples/flask/test_rock/app.py b/examples/flask/test_rock/app.py index dcefa1b..5fecffd 100644 --- a/examples/flask/test_rock/app.py +++ b/examples/flask/test_rock/app.py @@ -8,6 +8,7 @@ import boto3 import botocore.config +import pika import psycopg import pymongo import pymongo.database @@ -73,6 +74,22 @@ def get_redis_database() -> redis.Redis | None: return g.redis_db +def get_rabbitmq_connection(): + """Get rabbitmq connection.""" + if "rabbitmq" not in g: + if "RABBITMQ_HOSTNAME" in os.environ: + username = os.environ["RABBITMQ_USERNAME"] + password = os.environ["RABBITMQ_PASSWORD"] + hostname = os.environ["RABBITMQ_HOSTNAME"] + vhost = os.environ["RABBITMQ_VHOST"] + credentials = pika.PlainCredentials(username, password) + parameters = pika.ConnectionParameters(hostname, 5672, vhost, credentials) + g.rabbitmq = pika.BlockingConnection(parameters) + else: + return None + return g.rabbitmq + + def get_boto3_client(): if "boto3_client" not in g: if "S3_ACCESS_KEY" in os.environ: @@ -187,6 +204,31 @@ def redis_status(): return "FAIL" +@app.route("/rabbitmq/send") +def rabbitmq_send(): + """Send a message to "charm" queue.""" + if connection := get_rabbitmq_connection(): + channel = connection.channel() + channel.queue_declare(queue="charm") + channel.basic_publish(exchange="", routing_key="charm", body="SUCCESS") + return "SUCCESS" + return "FAIL" + + +@app.route("/rabbitmq/receive") +def rabbitmq_receive(): + """Receive a message from "charm" queue in blocking form.""" + if connection := get_rabbitmq_connection(): + channel = connection.channel() + method_frame, _header_frame, body = channel.basic_get("charm") + if method_frame: + channel.basic_ack(method_frame.delivery_tag) + if body == b"SUCCESS": + return "SUCCESS" + return "FAIL. INCORRECT MESSAGE." + return "FAIL. NO CONNECTION." + + @app.route("/env") def get_env(): """Return environment variables""" diff --git a/examples/flask/test_rock/requirements.txt b/examples/flask/test_rock/requirements.txt index 4c2df06..5f1ab38 100644 --- a/examples/flask/test_rock/requirements.txt +++ b/examples/flask/test_rock/requirements.txt @@ -6,3 +6,4 @@ psycopg[binary] pymongo redis[hiredis] boto3 +pika diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 34001dc..d50e955 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -15,32 +15,31 @@ logger = logging.getLogger(__name__) -@pytest_asyncio.fixture(scope="module", name="lxd_controller") -async def fixture_lxd_controller(ops_test: OpsTest) -> Controller: - """Return the current testing juju model.""" +@pytest_asyncio.fixture(scope="module", name="ops_test_lxd") +async def ops_test_lxd_fixture(request, tmp_path_factory): + # Create lxd controller if it does not exist if not "lxd" in Juju().get_controllers(): jujudata = FileJujuData() previous_controller = jujudata.current_controller() logger.info("bootstrapping lxd") _, _, _ = await ops_test.juju("bootstrap", "localhost", "lxd", check=True) # go back to the original controller - logger.info("switch back to %s", previous_controller) + logger.info("switch back to controller: %s", previous_controller) _, _, _ = await ops_test.juju("switch", previous_controller, check=True) - controller = Controller() - logger.info("connecting to lxd controller") - await controller.connect("lxd") - yield controller - logger.info("disconnecting from lxd controller") - await controller.disconnect() + ops_test = OpsTest(request, tmp_path_factory) + ops_test.controller_name = "lxd" + await ops_test._setup_model() + # The instance is not stored in _instance as that is done for the ops_test fixture + yield ops_test + await ops_test._cleanup_models() @pytest_asyncio.fixture(scope="module", name="lxd_model") -async def fixture_lxd_model(lxd_controller: Controller) -> Model: - model = await lxd_controller.add_model("lxd") - yield model - logger.info("destroy model %s", model.name) - # await lxd_controller.destroy_models(model.name, destroy_storage=True, force=True, max_wait=60) +async def lxd_model_fixture(ops_test_lxd: OpsTest) -> Model: + """Return the current lxd juju model.""" + assert ops_test_lxd.model + return ops_test_lxd.model @pytest_asyncio.fixture(scope="module", name="rabbitmq_server_app") # autouse=True) @@ -56,6 +55,22 @@ async def deploy_rabbitmq_server_fixture( await lxd_model.wait_for_idle(raise_on_blocked=True) offer = await lxd_model.create_offer("rabbitmq-server:amqp") logger.info("offer: %s", offer) + offers = await lxd_model.list_offers() + logger.info("offers: %s", offers) + yield app + + +@pytest_asyncio.fixture(scope="module", name="rabbitmq_k8s_app") # autouse=True) +async def deploy_rabbitmq_k8s_fixture( + model: Model, +) -> Application: + """Deploy rabbitmq-server machine app.""" + app = await model.deploy( + "rabbitmq-k8s", + channel="3.12/edge", + trust=True, + ) + await model.wait_for_idle(raise_on_blocked=True) yield app diff --git a/tests/integration/flask/conftest.py b/tests/integration/flask/conftest.py index 7eca1eb..9c83e5a 100644 --- a/tests/integration/flask/conftest.py +++ b/tests/integration/flask/conftest.py @@ -23,17 +23,6 @@ import asyncio -# import nest_asyncio - - -# @pytest.fixture(scope="module") -# def event_loop(): -# nest_asyncio.apply() -# loop = asyncio.new_event_loop() -# asyncio._set_running_loop(loop) -# yield loop -# loop.close() - @pytest.fixture(autouse=True) def cwd(): diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index 304b4b9..074a566 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -18,6 +18,7 @@ async def test_rabbitmq_server_integration( ops_test: OpsTest, + ops_test_lxd: OpsTest, flask_app: Application, rabbitmq_server_app: Application, model: Model, @@ -30,27 +31,58 @@ async def test_rabbitmq_server_integration( assert: TODO """ - # TODO GET THE OOFFER NAME FROM THE OFFER CREATION? - integration = await model.integrate("lxd:admin/lxd.rabbitmq-server", flask_app.name) - # integration = await model.integrate("rabbitmq-server", flask_app.name) + lxd_controller = await lxd_model.get_controller() + lxd_username = lxd_controller.get_current_username() + lxd_controller_name = ops_test_lxd.controller_name + lxd_model_name = lxd_model.name + offer_name = rabbitmq_server_app.name + rabbitmq_offer_url = f"{lxd_controller_name}:{lxd_username}/{lxd_model_name}.{offer_name}" - lxd_status = await lxd_model.get_status() - # get ip from lxd_status + integration = await model.integrate(rabbitmq_offer_url, flask_app.name) + await model.wait_for_idle(apps=[flask_app.name], status="active") - assert True + for unit_ip in await get_unit_ips(flask_app.name): + response = requests.get(f"http://{unit_ip}:8000/rabbitmq/send", timeout=5) + assert response.status_code == 200 + assert "SUCCESS" == response.text + + response = requests.get(f"http://{unit_ip}:8000/rabbitmq/receive", timeout=5) + assert response.status_code == 200 + assert "SUCCESS" == response.text + + res = await flask_app.destroy_relation("amqp", f"{lxd_model_name}:amqp") + await model.wait_for_idle(apps=[flask_app.name], status="active") + logger.info("destroy relation res %s", res) -# juju deploy rabbitmq-k8s --trust --channel=3.12/edge -# juju integrate flask-k8s rabbitmq-k8s + +async def test_rabbitmq_k8s_integration( + ops_test: OpsTest, + flask_app: Application, + rabbitmq_k8s_app: Application, + model: Model, + get_unit_ips, +): + """ + arrange: TODO + act: TODO + assert: TODO + """ + + integration = await model.integrate(rabbitmq_k8s_app.name, flask_app.name) + await model.wait_for_idle(apps=[flask_app.name], status="active") + + for unit_ip in await get_unit_ips(flask_app.name): + response = requests.get(f"http://{unit_ip}:8000/rabbitmq/send", timeout=5) + assert response.status_code == 200 + assert "SUCCESS" == response.text + + response = requests.get(f"http://{unit_ip}:8000/rabbitmq/receive", timeout=5) + assert response.status_code == 200 + assert "SUCCESS" == response.text -# juju bootstrap localhost lxd -# lxd: -# juju switch lxd -# juju add-model whatever -# juju deploy rabbitmq-server -# juju offer rabbitmq-server:amqp -# juju switch microk8s -# juju integrate lxd:admin/welcome-lxd.rabbitmq-server flask-k8s + res = await flask_app.destroy_relation("amqp", f"{rabbitmq_k8s_app.name}:amqp") + logger.info("destroy relation res %s", res) async def test_s3_integration( From 1dc37bbdae6b7f20cba836b75e8bfae1c7b1a46f Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 3 Sep 2024 14:35:55 +0200 Subject: [PATCH 09/40] fix linting --- tests/integration/flask/test_integrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index 074a566..68013be 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -55,7 +55,7 @@ async def test_rabbitmq_server_integration( logger.info("destroy relation res %s", res) - + async def test_rabbitmq_k8s_integration( ops_test: OpsTest, flask_app: Application, From 54690724632e3da445cac0b10b32a23b6b15b0b6 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 3 Sep 2024 14:38:04 +0200 Subject: [PATCH 10/40] remove tmate. more messages --- .github/workflows/integration_test.yaml | 1 - examples/flask/test_rock/app.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 5f760f1..0751bab 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -19,4 +19,3 @@ jobs: rockcraft-ref: fastapi-extension juju-channel: ${{ matrix.juju-version }} channel: 1.29-strict/stable - tmate-debug: true diff --git a/examples/flask/test_rock/app.py b/examples/flask/test_rock/app.py index 5fecffd..a928fe9 100644 --- a/examples/flask/test_rock/app.py +++ b/examples/flask/test_rock/app.py @@ -225,7 +225,8 @@ def rabbitmq_receive(): channel.basic_ack(method_frame.delivery_tag) if body == b"SUCCESS": return "SUCCESS" - return "FAIL. INCORRECT MESSAGE." + return "FAIL. INCORRECT MESSAGE." + return "FAIL. NO MESSAGE." return "FAIL. NO CONNECTION." From 962ad62414472de578e5c4c1ea75ea4857da1eb0 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 3 Sep 2024 16:28:55 +0200 Subject: [PATCH 11/40] add ops_test fixture --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d50e955..6a3b18d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -16,7 +16,7 @@ @pytest_asyncio.fixture(scope="module", name="ops_test_lxd") -async def ops_test_lxd_fixture(request, tmp_path_factory): +async def ops_test_lxd_fixture(request, tmp_path_factory, ops_test: OpsTest): # Create lxd controller if it does not exist if not "lxd" in Juju().get_controllers(): jujudata = FileJujuData() From d2bcc3708b247503aecd54509ffdcd1f40e79a43 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 3 Sep 2024 17:11:31 +0200 Subject: [PATCH 12/40] do not switch controller --- tests/integration/conftest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6a3b18d..0edbb82 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -21,11 +21,12 @@ async def ops_test_lxd_fixture(request, tmp_path_factory, ops_test: OpsTest): if not "lxd" in Juju().get_controllers(): jujudata = FileJujuData() previous_controller = jujudata.current_controller() - logger.info("bootstrapping lxd") + logger.info("JAVI controller %s", jujudata.current_controller()) + logger.info("bootstrapping lxd")q _, _, _ = await ops_test.juju("bootstrap", "localhost", "lxd", check=True) - # go back to the original controller - logger.info("switch back to controller: %s", previous_controller) - _, _, _ = await ops_test.juju("switch", previous_controller, check=True) + jujudata = FileJujuData() + previous_controller = jujudata.current_controller() + logger.info("JAVI controller %s", jujudata.current_controller()) ops_test = OpsTest(request, tmp_path_factory) ops_test.controller_name = "lxd" From a3b9c87af45949a1968847971d83bbbad5518ebb Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 3 Sep 2024 17:15:35 +0200 Subject: [PATCH 13/40] fix error --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0edbb82..7bf4245 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -22,7 +22,7 @@ async def ops_test_lxd_fixture(request, tmp_path_factory, ops_test: OpsTest): jujudata = FileJujuData() previous_controller = jujudata.current_controller() logger.info("JAVI controller %s", jujudata.current_controller()) - logger.info("bootstrapping lxd")q + logger.info("bootstrapping lxd") _, _, _ = await ops_test.juju("bootstrap", "localhost", "lxd", check=True) jujudata = FileJujuData() previous_controller = jujudata.current_controller() From 4a243650feab662b3bdae24fbb7e1e7a04a3bf4b Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 3 Sep 2024 17:53:34 +0200 Subject: [PATCH 14/40] Add some logs --- tests/integration/flask/test_integrations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index 68013be..ab0e235 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -50,6 +50,9 @@ async def test_rabbitmq_server_integration( assert response.status_code == 200 assert "SUCCESS" == response.text + status = await model.get_status() + logger.info("status: %s", status) + logger.info("destroying: %s - %s", "amqp", f"{lxd_model_name}:amqp") res = await flask_app.destroy_relation("amqp", f"{lxd_model_name}:amqp") await model.wait_for_idle(apps=[flask_app.name], status="active") From 474fc198a6faa21f326caf2167abfdfe6396f6b8 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 3 Sep 2024 18:19:45 +0200 Subject: [PATCH 15/40] update name to destroy relation --- tests/integration/flask/test_integrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index ab0e235..dfc334b 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -53,7 +53,7 @@ async def test_rabbitmq_server_integration( status = await model.get_status() logger.info("status: %s", status) logger.info("destroying: %s - %s", "amqp", f"{lxd_model_name}:amqp") - res = await flask_app.destroy_relation("amqp", f"{lxd_model_name}:amqp") + res = await flask_app.destroy_relation("amqp", f"{rabbitmq_server_app.name}:amqp") await model.wait_for_idle(apps=[flask_app.name], status="active") logger.info("destroy relation res %s", res) From 47f1997f10c4fd34ff662216dfa2e22dba6a0e4b Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 4 Sep 2024 11:55:52 +0200 Subject: [PATCH 16/40] Starting to clean the relation and the charm --- paas_app_charmer/charm.py | 8 +- paas_app_charmer/rabbitmq.py | 160 ++++++++++++++++++++++------------- 2 files changed, 100 insertions(+), 68 deletions(-) diff --git a/paas_app_charmer/charm.py b/paas_app_charmer/charm.py index 37ee9a4..c711720 100644 --- a/paas_app_charmer/charm.py +++ b/paas_app_charmer/charm.py @@ -104,18 +104,12 @@ def __init__(self, framework: ops.Framework, framework_name: str) -> None: self._amqp: RabbitMQRequires | None if "amqp" in requires and requires["amqp"].interface_name == "rabbitmq": - logger.info("JAVI PREPARING RABBIT REQUIRES") self._amqp = RabbitMQRequires( self, "amqp", username=self.app.name, vhost="/", ) - logger.info("JAVI rabbitmq_requires._amqp_rel %s", str(self._amqp._amqp_rel)) - if self._amqp._amqp_rel is not None: - logger.info( - "JAVI rabbitmq_requires._amqp_rel.data %s", str(self._amqp._amqp_rel.data) - ) self.framework.observe(self._amqp.on.connected, self._on_amqp_connected) self.framework.observe(self._amqp.on.ready, self._on_amqp_ready) self.framework.observe(self._amqp.on.goneaway, self._on_amqp_goneaway) @@ -266,7 +260,7 @@ def is_ready(self) -> bool: return True - # JAVI PENDING TO REFACTOR + # Pending to refactor all integrations def _missing_required_integrations(self, charm_state: CharmState) -> list[str]: # noqa: C901 """Get list of missing integrations that are required. diff --git a/paas_app_charmer/rabbitmq.py b/paas_app_charmer/rabbitmq.py index 5ae5bc9..4302c86 100644 --- a/paas_app_charmer/rabbitmq.py +++ b/paas_app_charmer/rabbitmq.py @@ -14,26 +14,28 @@ part in the relation bag. This python library unifies both charms, using a similar approach to the rabbitmq-k8s library. -# rabbitmq-k8s -# unit-flask-k8s-0: 15:27:43 INFO unit.flask-k8s/0.juju-log JAVI rabbitmq_requires._amqp_rel.data {: {'egress-subnets': '10.152.183.168/32', 'ingress-address': '10.152.183.168', 'private-address': '10.152.183.168'}, : {'username': 'flask-k8s', 'vhost': '/'}, : {'egress-subnets': '10.12.97.225/32', 'ingress-address': '10.12.97.225', 'private-address': '10.12.97.225'}, : {'egress-subnets': '10.12.97.225/32', 'ingress-address': '10.12.97.225', 'private-address': '10.12.97.225'}, : {'hostname': 'rabbitmq-k8s-endpoints.testing.svc.cluster.local', 'password': '3m036hhyiDHs'}} # noqa: W505 # pylint: disable=line-too-long +For rabbitmq-k8s, the password and hostname are required in the app databag. The full +list of hostnames can be obtained from the ingress-address in each unit. -# rabbitmq-server -# unit-flask-k8s-0: 16:00:16 INFO unit.flask-k8s/0.juju-log amqp:3: JAVI rabbitmq_requires._amqp_rel.data {: {'egress-subnets': '10.152.183.168/32', 'ingress-address': '10.152.183.168', 'private-address': '10.152.183.168'}, : {'username': 'flask-k8s', 'vhost': '/'}, : {'hostname': '10.58.171.158', 'password': 'LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8', 'private-address': '10.58.171.158'}, : {'hostname': '10.58.171.70', 'password': 'LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8', 'private-address': '10.58.171.70'}, : {'egress-subnets': '10.58.171.146/32', 'hostname': '10.58.171.146', 'ingress-address': '10.58.171.146', 'password': 'LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8', 'private-address': '10.58.171.146'}, : {}} +For rabbitmq-server, the app databag is empty. The password and hostname are in the units databags, +being the password equal in all units. Each hostname may point to different addresses. One +of them will chosen as the in the rabbitmq parameters. + +rabbitmq-server support ssl client certificates, but are not implemented in this library. + +This library is very similar and uses the same events as + the library charms.rabbitmq_k8s.v0.rabbitmq. +See https://github.com/openstack-charmers/charm-rabbitmq-k8s/blob/main/lib/charms/rabbitmq_k8s/v0/rabbitmq.py # pylint: disable=line-too-long # noqa: W505 """ -# JAVI COPYPASTED FOR NOW! -# JAVI EXPLAIN NO SSL CLIENT CERFICATES YET import logging -from typing import cast from ops import CharmBase, HookEvent from ops.framework import EventBase, EventSource, Object, ObjectEvents from ops.model import Relation -from pydantic import ValidationError from paas_app_charmer.charm_state import RabbitMQParameters -from paas_app_charmer.utils import build_validation_error_message logger = logging.getLogger(__name__) @@ -54,9 +56,9 @@ class RabbitMQServerEvents(ObjectEvents): """Events class for `on`. Attributes: - connected: JAVI TODO - ready: JAVI TODO - goneaway: JAVI TODO + connected: rabbitmq relation is connected + ready: rabbitmq relation is ready + goneaway: rabbitmq relation has been removed """ connected = EventSource(RabbitMQConnectedEvent) @@ -68,19 +70,19 @@ class RabbitMQRequires(Object): """RabbitMQRequires class. Attributes: - on: JAVI TODO + on: ObjectEvents for RabbitMQRequires """ on = RabbitMQServerEvents() def __init__(self, charm: CharmBase, relation_name: str, username: str, vhost: str): - """JAVI TODO. + """Initialize the instance. Args: - charm: JAVI TODO - relation_name: JAVI TODO - username: JAVI TODO - vhost: JAVI TODO + charm: charm that uses the library. + relation_name: name of the RabbitMQ relation + username: username to use for RabbitMQ + vhost: virtual host to use for RabbitMQ """ super().__init__(charm, relation_name) self.charm = charm @@ -105,20 +107,17 @@ def __init__(self, charm: CharmBase, relation_name: str, username: str, vhost: s ) def _on_amqp_relation_joined(self, _: HookEvent) -> None: - """JAVI TODO.""" - logging.debug("RabbitMQRabbitMQRequires on_joined") + """Handle RabbitMQ joined.""" self.on.connected.emit() self.request_access(self.username, self.vhost) def _on_amqp_relation_changed(self, _: HookEvent) -> None: - """JAVI TODO.""" - logging.debug("RabbitMQRabbitMQRequires on_changed/departed") + """Handle RabbitMQ changed.""" if self.rabbitmq_parameters(): self.on.ready.emit() def _on_amqp_relation_broken(self, _: HookEvent) -> None: - """JAVI TODO.""" - logging.debug("RabbitMQRabbitMQRequires on_broken") + """Handle RabbitMQ broken.""" self.on.goneaway.emit() @property @@ -127,55 +126,94 @@ def _amqp_rel(self) -> Relation | None: return self.framework.model.get_relation(self.relation_name) def rabbitmq_parameters(self) -> RabbitMQParameters | None: - """TODO JAVI. + """Return RabbitMQ parameters with the data in the relation. + + It will try to use the format in rabbitmq-k8s or rabbitmq-server. + If there is no relation or the data is not complete, it returns None. Returns: - TODO JAVI + The parameters for RabbitMQ or None. """ - if self._amqp_rel: - # just put everything. - hostnames = [] - password = None - for unit in self._amqp_rel.units: - unit_data = self._amqp_rel.data[unit] - ingress_address = unit_data.get("ingress-address") - unit_hostname = unit_data.get("hostname", ingress_address) - if unit_hostname: - hostnames.append(unit_hostname) - # all of them should be equal - password = unit_data.get("password", password) - - password = self._amqp_rel.data[self._amqp_rel.app].get("password", password) - first_hostname = hostnames[0] if len(hostnames) > 0 else None - hostname = self._amqp_rel.data[self._amqp_rel.app].get("hostname", first_hostname) - - try: - return RabbitMQParameters( - hostname=cast(str, hostname), - hostnames=hostnames, - username=self.username, - password=cast(str, password), - vhost=self.vhost, - ) - except ValidationError as exc: - # do not crash, as the relation can be starting at this moment, and the - # data not being there in the current hook - error_message = build_validation_error_message(exc) - logger.info("Error validating RabbitMQ parameters %s", error_message) - return None - return None + rabbitmq_k8s_params = self._rabbitmq_k8s_parameters() + if rabbitmq_k8s_params: + return rabbitmq_k8s_params + + # rabbitmq-server parameters or None. + return self._rabbitmq_server_parameters() def request_access(self, username: str, vhost: str) -> None: """Request access to the RabbitMQ server. Args: - username: JAVI TODO - vhost: JAVI TODO + username: username requested for RabbitMQ + vhost: virtual host requested for RabbitMQ """ if self.model.unit.is_leader(): - logging.debug("Requesting RabbitMQ user and vhost") if self._amqp_rel: self._amqp_rel.data[self.charm.app]["username"] = username self._amqp_rel.data[self.charm.app]["vhost"] = vhost else: logger.warning("request_access but no rabbitmq relation") + + def _rabbitmq_server_parameters(self) -> RabbitMQParameters | None: + """Return parameters for rabbitmq-server. + + Returns: + Returns parameters for rabbitmq-server or None if they are not valid/complete. + """ + if not self._amqp_rel: + return None + + password = None + hostnames = [] + for unit in self._amqp_rel.units: + unit_data = self._amqp_rel.data[unit] + # All of the passwords should be equal. If it is + # in the unit data, get it and override the password + password = unit_data.get("password", password) + unit_hostname = unit_data.get("hostname") + if unit_hostname: + hostnames.append(unit_hostname) + + if not password or len(hostnames) == 0: + return None + + hostname = hostnames[0] + return RabbitMQParameters( + hostname=hostname, + hostnames=hostnames, + username=self.username, + password=password, + vhost=self.vhost, + ) + + def _rabbitmq_k8s_parameters(self) -> RabbitMQParameters | None: + """Return parameters for rabbitmq-k8s. + + Returns: + Returns parameters for rabbitmq-k8s or None if they are not valid/complete. + """ + if not self._amqp_rel: + return None + + # A password in the _amqp_rel data differentiates rabbitmq-k8s from rabbitmq-server + password = self._amqp_rel.data[self._amqp_rel.app].get("password") + hostname = self._amqp_rel.data[self._amqp_rel.app].get("hostname") + + hostnames = [] + for unit in self._amqp_rel.units: + unit_data = self._amqp_rel.data[unit] + ingress_address = unit_data.get("ingress-address") + if ingress_address: + hostnames.append(ingress_address) + + if not password or not hostname: + return None + + return RabbitMQParameters( + hostname=hostname, + hostnames=hostnames, + username=self.username, + password=password, + vhost=self.vhost, + ) From ef932b137ee80991ac41f546352babb13b9b1050 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 4 Sep 2024 11:56:37 +0200 Subject: [PATCH 17/40] remove unnecesasry logging --- paas_app_charmer/charm.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/paas_app_charmer/charm.py b/paas_app_charmer/charm.py index c711720..355e9a0 100644 --- a/paas_app_charmer/charm.py +++ b/paas_app_charmer/charm.py @@ -442,17 +442,14 @@ def _on_pebble_ready(self, _: ops.PebbleReadyEvent) -> None: @block_if_invalid_config def _on_amqp_connected(self, _event: ops.HookEvent) -> None: """Handle ampq connected event.""" - logger.info("_ON_AMQP_CONNECTED") self.restart() @block_if_invalid_config def _on_amqp_ready(self, _event: ops.HookEvent) -> None: """Handle ampq ready event.""" - logger.info("_ON_AMQP_READY") self.restart() @block_if_invalid_config def _on_amqp_goneaway(self, _event: ops.HookEvent) -> None: """Handle ampq goneaway event.""" - logger.info("_ON_AMQP_GONEAWAY") self.restart() From fe2a38e66aca00d12e8176668d21f214a38fd5e3 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 4 Sep 2024 11:58:32 +0200 Subject: [PATCH 18/40] improve comments --- paas_app_charmer/charm_state.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/paas_app_charmer/charm_state.py b/paas_app_charmer/charm_state.py index 47723f8..db01c0c 100644 --- a/paas_app_charmer/charm_state.py +++ b/paas_app_charmer/charm_state.py @@ -102,7 +102,7 @@ def from_charm( # pylint: disable=too-many-arguments redis_uri: The redis uri provided by the redis charm. s3_connection_info: Connection info from S3 lib. saml_relation_data: Relation data from the SAML app. - rabbitmq_parameters: JAVI TODO. + rabbitmq_parameters: RabbitMQ parameters. base_url: Base URL for the service. Return: @@ -235,7 +235,7 @@ def build( # pylint: disable=too-many-arguments database_requirers: All database requirers object declared by the charm. s3_connection_info: S3 connection info from S3 lib. saml_relation_data: Saml relation data from saml lib. - rabbitmq_parameters: TODO JAVI + rabbitmq_parameters: RabbitMQ parameters. Return: The IntegrationsState instance created. @@ -368,18 +368,14 @@ class RabbitMQParameters(BaseModel, extra=Extra.allow): """Configuration for accessing RabbitMQ. Attributes: - hostname: Hostname or ip address of broker. It is one of the hostnames. + hostname: Hostname or ip address of broker. It could be one of the hostnames. hostnames: List of Hostnames or ip addresses of broker. username: The username to authenticate with password: The password to authenticate with vhost: RabbitMQ virtual host name. """ - # In rabbitmq-k8s, it is the service url - # In rabbitmq-server, paas-app-charmer puts one of the ips of hosnames arbitrarily hostname: str - # In rabbitmq-k8s, a list of k8s ingress-address (maybe the same one) - # In rabbitmq-server, the ips of the machines hostnames: list[str] username: str password: str From 8c4565bac5767329df09f4b31e5f041a0db7ace5 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 4 Sep 2024 13:02:20 +0200 Subject: [PATCH 19/40] Some more cleaning --- tests/integration/conftest.py | 11 +--- tests/integration/flask/conftest.py | 44 +++++++++++++- tests/integration/flask/test_integrations.py | 60 +++++--------------- tests/unit/flask/test_charm.py | 43 +++++++++++++- tests/unit/go/test_app.py | 17 +++++- 5 files changed, 114 insertions(+), 61 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7bf4245..e0cd379 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -17,16 +17,10 @@ @pytest_asyncio.fixture(scope="module", name="ops_test_lxd") async def ops_test_lxd_fixture(request, tmp_path_factory, ops_test: OpsTest): - # Create lxd controller if it does not exist + """Return a ops_test fixture for lxd, creating the lxd controller if it does not exist.""" if not "lxd" in Juju().get_controllers(): - jujudata = FileJujuData() - previous_controller = jujudata.current_controller() - logger.info("JAVI controller %s", jujudata.current_controller()) logger.info("bootstrapping lxd") _, _, _ = await ops_test.juju("bootstrap", "localhost", "lxd", check=True) - jujudata = FileJujuData() - previous_controller = jujudata.current_controller() - logger.info("JAVI controller %s", jujudata.current_controller()) ops_test = OpsTest(request, tmp_path_factory) ops_test.controller_name = "lxd" @@ -50,7 +44,6 @@ async def deploy_rabbitmq_server_fixture( """Deploy rabbitmq-server machine app.""" app = await lxd_model.deploy( "rabbitmq-server", - # channel="3.9/stable", channel="latest/edge", ) await lxd_model.wait_for_idle(raise_on_blocked=True) @@ -65,7 +58,7 @@ async def deploy_rabbitmq_server_fixture( async def deploy_rabbitmq_k8s_fixture( model: Model, ) -> Application: - """Deploy rabbitmq-server machine app.""" + """Deploy rabbitmq-k8s app.""" app = await model.deploy( "rabbitmq-k8s", channel="3.12/edge", diff --git a/tests/integration/flask/conftest.py b/tests/integration/flask/conftest.py index 9c83e5a..e8a838b 100644 --- a/tests/integration/flask/conftest.py +++ b/tests/integration/flask/conftest.py @@ -21,9 +21,6 @@ PROJECT_ROOT = pathlib.Path(__file__).parent.parent.parent.parent -import asyncio - - @pytest.fixture(autouse=True) def cwd(): return os.chdir(PROJECT_ROOT / "examples/flask") @@ -255,3 +252,44 @@ def boto_s3_client_fixture(model: Model, s3_configuration: dict, s3_credentials: config=s3_client_config, ) yield s3_client + + +@pytest_asyncio.fixture(scope="function", name="rabbitmq_server_integration") +async def rabbitmq_server_integration_fixture( + ops_test_lxd: OpsTest, + flask_app: Application, + rabbitmq_server_app: Application, + model: Model, + lxd_model: Model, +): + """Integrates flask with rabbitmq-server.""" + lxd_controller = await lxd_model.get_controller() + lxd_username = lxd_controller.get_current_username() + lxd_controller_name = ops_test_lxd.controller_name + lxd_model_name = lxd_model.name + offer_name = rabbitmq_server_app.name + rabbitmq_offer_url = f"{lxd_controller_name}:{lxd_username}/{lxd_model_name}.{offer_name}" + + integration = await model.integrate(rabbitmq_offer_url, flask_app.name) + await model.wait_for_idle(apps=[flask_app.name], status="active") + + yield integration + + res = await flask_app.destroy_relation("amqp", f"{rabbitmq_server_app.name}:amqp") + await model.wait_for_idle(apps=[flask_app.name], status="active") + + +@pytest_asyncio.fixture(scope="function", name="rabbitmq_k8s_integration") +async def rabbitmq_k8s_integration_fixture( + model: Model, + rabbitmq_k8s_app: Application, + flask_app: Application, +): + """Integrates flask with rabbitmq-k8s.""" + integration = await model.integrate(rabbitmq_k8s_app.name, flask_app.name) + await model.wait_for_idle(apps=[flask_app.name], status="active") + + yield integration + + await flask_app.destroy_relation("amqp", f"{rabbitmq_k8s_app.name}:amqp") + await model.wait_for_idle(apps=[flask_app.name], status="active") diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index dfc334b..4b343fe 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -7,6 +7,7 @@ from secrets import token_hex import ops +import pytest import requests from juju.application import Application from juju.model import Model @@ -16,65 +17,35 @@ logger = logging.getLogger(__name__) +@pytest.mark.usefixtures("rabbitmq_server_integration") async def test_rabbitmq_server_integration( - ops_test: OpsTest, - ops_test_lxd: OpsTest, flask_app: Application, - rabbitmq_server_app: Application, - model: Model, - lxd_model: Model, get_unit_ips, ): """ - arrange: TODO - act: TODO - assert: TODO + arrange: Flask and rabbitmq-server deployed + act: Integrate flask with rabbitmq-server + assert: Assert that RabbitMQ works correctly """ - - lxd_controller = await lxd_model.get_controller() - lxd_username = lxd_controller.get_current_username() - lxd_controller_name = ops_test_lxd.controller_name - lxd_model_name = lxd_model.name - offer_name = rabbitmq_server_app.name - rabbitmq_offer_url = f"{lxd_controller_name}:{lxd_username}/{lxd_model_name}.{offer_name}" - - integration = await model.integrate(rabbitmq_offer_url, flask_app.name) - await model.wait_for_idle(apps=[flask_app.name], status="active") - - for unit_ip in await get_unit_ips(flask_app.name): - response = requests.get(f"http://{unit_ip}:8000/rabbitmq/send", timeout=5) - assert response.status_code == 200 - assert "SUCCESS" == response.text - - response = requests.get(f"http://{unit_ip}:8000/rabbitmq/receive", timeout=5) - assert response.status_code == 200 - assert "SUCCESS" == response.text - - status = await model.get_status() - logger.info("status: %s", status) - logger.info("destroying: %s - %s", "amqp", f"{lxd_model_name}:amqp") - res = await flask_app.destroy_relation("amqp", f"{rabbitmq_server_app.name}:amqp") - await model.wait_for_idle(apps=[flask_app.name], status="active") - - logger.info("destroy relation res %s", res) + await assert_rabbitmq_integration_correct(flask_app, get_unit_ips) +@pytest.mark.usefixtures("rabbitmq_k8s_integration") async def test_rabbitmq_k8s_integration( - ops_test: OpsTest, flask_app: Application, - rabbitmq_k8s_app: Application, - model: Model, get_unit_ips, ): """ - arrange: TODO - act: TODO - assert: TODO + arrange: Flask and rabbitmq-k8s deployed + act: Integrate flask with rabbitmq-k8s + assert: Assert that RabbitMQ works correctly + """ + await assert_rabbitmq_integration_correct(flask_app, get_unit_ips) - integration = await model.integrate(rabbitmq_k8s_app.name, flask_app.name) - await model.wait_for_idle(apps=[flask_app.name], status="active") +async def assert_rabbitmq_integration_correct(flask_app: Application, get_unit_ips): + """Assert that rabbitmq works correctly sending and receiving a message.""" for unit_ip in await get_unit_ips(flask_app.name): response = requests.get(f"http://{unit_ip}:8000/rabbitmq/send", timeout=5) assert response.status_code == 200 @@ -84,9 +55,6 @@ async def test_rabbitmq_k8s_integration( assert response.status_code == 200 assert "SUCCESS" == response.text - res = await flask_app.destroy_relation("amqp", f"{rabbitmq_k8s_app.name}:amqp") - logger.info("destroy relation res %s", res) - async def test_s3_integration( ops_test: OpsTest, diff --git a/tests/unit/flask/test_charm.py b/tests/unit/flask/test_charm.py index 6feea23..9aa64e5 100644 --- a/tests/unit/flask/test_charm.py +++ b/tests/unit/flask/test_charm.py @@ -213,11 +213,12 @@ def test_rabbitmq_integration(harness: Harness, amqp_relation_data, expected_env assert: The flask service should have environment variables in its plan for each of the integrations. """ - harness.add_relation("amqp", "amqp", **amqp_relation_data) container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) container.add_layer("a_layer", DEFAULT_LAYER) + harness.begin_with_initial_hooks() + assert harness.model.unit.status == ops.ActiveStatus() service_env = container.get_plan().services["flask"].environment for env, env_val in expected_env_vars.items(): @@ -225,6 +226,46 @@ def test_rabbitmq_integration(harness: Harness, amqp_relation_data, expected_env assert service_env[env] == env_val +def test_rabbitmq_integration_with_relation_data_empty(harness: Harness): + """ + arrange: Prepare a amqp integration (RabbitMQ), with missing data. + act: Start the flask charm and set flask-app container to be ready. + assert: The flask service should not have environment variables related to RabbitMQ + """ + harness.add_relation("amqp", "amqp") + container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) + container.add_layer("a_layer", DEFAULT_LAYER) + + harness.begin_with_initial_hooks() + + assert harness.model.unit.status == ops.ActiveStatus() + service_env = container.get_plan().services["flask"].environment + for env in service_env.keys(): + assert "RABBITMQ" not in env + + +def test_rabbitmq_remove_integration(harness: Harness): + """ + arrange: Prepare a charm with a complete amqp integration (RabbitMQ). + act: Remove the relation. + assert: The relation should not have the env variables related to RabbitMQ. + """ + relation_id = harness.add_relation( + "amqp", "amqp", app_data={"hostname": "example.com", "password": "p"} + ) + container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) + container.add_layer("a_layer", DEFAULT_LAYER) + harness.begin_with_initial_hooks() + assert harness.model.unit.status == ops.ActiveStatus() + service_env = container.get_plan().services["flask"].environment + assert "RABBITMQ_HOSTNAME" in service_env + + harness.remove_relation(relation_id) + + service_env = container.get_plan().services["flask"].environment + assert "RABBITMQ_HOSTNAME" not in service_env + + @pytest.mark.parametrize( "integrate_to,required_integrations", [ diff --git a/tests/unit/go/test_app.py b/tests/unit/go/test_app.py index 1cd9d75..bd0a4fd 100644 --- a/tests/unit/go/test_app.py +++ b/tests/unit/go/test_app.py @@ -9,7 +9,7 @@ import pytest from paas_app_charmer.app import App, WorkloadConfig -from paas_app_charmer.charm_state import CharmState, IntegrationsState +from paas_app_charmer.charm_state import CharmState, IntegrationsState, RabbitMQParameters from paas_app_charmer.go.charm import GoConfig @@ -32,7 +32,16 @@ {"JUJU_CHARM_HTTP_PROXY": "http://proxy.test"}, {"extra-config", "extravalue"}, {"metrics-port": "9000", "metrics-path": "/m", "app-secret-key": "notfoobar"}, - IntegrationsState(redis_uri="redis://10.1.88.132:6379"), + IntegrationsState( + redis_uri="redis://10.1.88.132:6379", + rabbitmq_parameters=RabbitMQParameters( + hostname="rabbitmq.example.com", + hostnames=["127.0.0.1"], + username="go-app", + password="nothingspecial", + vhost="./", + ), + ), { "APP_PORT": "8080", "APP_METRICS_PATH": "/m", @@ -51,6 +60,10 @@ "APP_REDIS_DB_PORT": "6379", "APP_REDIS_DB_QUERY": "", "APP_REDIS_DB_SCHEME": "redis", + "APP_RABBITMQ_HOSTNAME": "rabbitmq.example.com", + "APP_RABBITMQ_PASSWORD": "nothingspecial", + "APP_RABBITMQ_USERNAME": "go-app", + "APP_RABBITMQ_VHOST": "./", }, ), ], From 93c511e46376e1c476eeb68d20bb8aa8af8953b5 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 4 Sep 2024 13:05:09 +0200 Subject: [PATCH 20/40] remove unneded logs and listing of offers --- paas_app_charmer/rabbitmq.py | 2 +- tests/integration/conftest.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/paas_app_charmer/rabbitmq.py b/paas_app_charmer/rabbitmq.py index 4302c86..666083d 100644 --- a/paas_app_charmer/rabbitmq.py +++ b/paas_app_charmer/rabbitmq.py @@ -79,7 +79,7 @@ def __init__(self, charm: CharmBase, relation_name: str, username: str, vhost: s """Initialize the instance. Args: - charm: charm that uses the library. + charm: charm that uses the library relation_name: name of the RabbitMQ relation username: username to use for RabbitMQ vhost: virtual host to use for RabbitMQ diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e0cd379..efb80af 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -47,10 +47,7 @@ async def deploy_rabbitmq_server_fixture( channel="latest/edge", ) await lxd_model.wait_for_idle(raise_on_blocked=True) - offer = await lxd_model.create_offer("rabbitmq-server:amqp") - logger.info("offer: %s", offer) - offers = await lxd_model.list_offers() - logger.info("offers: %s", offers) + await lxd_model.create_offer("rabbitmq-server:amqp") yield app From df8f391a45fa104fe3a00707d2d8099edc6c6bf8 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Thu, 5 Sep 2024 16:50:17 +0200 Subject: [PATCH 21/40] rename relation from amqp to rabbitmq --- examples/flask/charmcraft.yaml | 2 +- paas_app_charmer/__init__.py | 24 ++++++++--------- paas_app_charmer/charm.py | 42 +++++++++++++---------------- paas_app_charmer/rabbitmq.py | 40 +++++++++++++-------------- tests/integration/flask/conftest.py | 4 +-- tests/unit/flask/test_charm.py | 20 +++++++------- 6 files changed, 65 insertions(+), 67 deletions(-) diff --git a/examples/flask/charmcraft.yaml b/examples/flask/charmcraft.yaml index ae92bf5..ca7894f 100644 --- a/examples/flask/charmcraft.yaml +++ b/examples/flask/charmcraft.yaml @@ -115,7 +115,7 @@ requires: interface: saml optional: True limit: 1 - amqp: + rabbitmq: interface: rabbitmq optional: True limit: 1 diff --git a/paas_app_charmer/__init__.py b/paas_app_charmer/__init__.py index 570e36d..03d0797 100644 --- a/paas_app_charmer/__init__.py +++ b/paas_app_charmer/__init__.py @@ -12,46 +12,46 @@ import charms.traefik_k8s.v2.ingress # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run `charmcraft fetch-lib charms.traefik_k8s.v2.ingress`" + "Missing charm library, please run `charmcraft fetch-libs`" ) from import_error try: import charms.observability_libs.v0.juju_topology # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run " - "`charmcraft fetch-lib charms.observability_libs.v0.juju_topology`" + "Missing charm library, please run `charmcraft fetch-libs`" + "`charmcraft fetch-libs`" ) from import_error try: import charms.grafana_k8s.v0.grafana_dashboard # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run " - "`charmcraft fetch-lib charms.grafana_k8s.v0.grafana_dashboard`" + "Missing charm library, please run `charmcraft fetch-libs`" + "`charmcraft fetch-libs`" ) from import_error try: import charms.loki_k8s.v0.loki_push_api # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run " - "`charmcraft fetch-lib charms.loki_k8s.v0.loki_push_api`" + "Missing charm library, please run `charmcraft fetch-libs`" + "`charmcraft fetch-libs`" ) from import_error try: import charms.prometheus_k8s.v0.prometheus_scrape # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run " - "`charmcraft fetch-lib charms.prometheus_k8s.v0.prometheus_scrape`" + "Missing charm library, please run `charmcraft fetch-libs`" + "`charmcraft fetch-libs`" ) from import_error try: import charms.data_platform_libs.v0.data_interfaces # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run " - "`charmcraft fetch-lib charms.data_platform_libs.v0.data_interfaces`" + "Missing charm library, please run `charmcraft fetch-libs`" + "`charmcraft fetch-libs`" ) from import_error try: import charms.redis_k8s.v0.redis # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run `charmcraft fetch-lib charms.redis_k8s.v0.redis`" + "Missing charm library, please run `charmcraft fetch-libs`" ) from import_error diff --git a/paas_app_charmer/charm.py b/paas_app_charmer/charm.py index 355e9a0..8f0f13c 100644 --- a/paas_app_charmer/charm.py +++ b/paas_app_charmer/charm.py @@ -25,23 +25,19 @@ logger = logging.getLogger(__name__) -# Until charmcraft fetch-libs is implemented, the charm will not fail -# if new optional libs are not fetched, as it will not be backwards compatible. +# Do not fail for new integrations to be backwards compatible without forcing +# the user to run fetch-libs. try: # pylint: disable=ungrouped-imports from charms.data_platform_libs.v0.s3 import S3Requirer except ImportError: - logger.exception( - "Missing charm library, please run `charmcraft fetch-lib charms.data_platform_libs.v0.s3`" - ) + logger.exception("Missing charm library, please run `charmcraft fetch-libs`") try: # pylint: disable=ungrouped-imports from charms.saml_integrator.v0.saml import SamlRequires except ImportError: - logger.exception( - "Missing charm library, please run `charmcraft fetch-lib charms.saml_integrator.v0.saml`" - ) + logger.exception("Missing charm library, please run `charmcraft fetch-libs`") class PaasCharm(abc.ABC, ops.CharmBase): # pylint: disable=too-many-instance-attributes @@ -102,19 +98,19 @@ def __init__(self, framework: ops.Framework, framework_name: str) -> None: else: self._saml = None - self._amqp: RabbitMQRequires | None - if "amqp" in requires and requires["amqp"].interface_name == "rabbitmq": - self._amqp = RabbitMQRequires( + self._rabbitmq: RabbitMQRequires | None + if "rabbitmq" in requires and requires["rabbitmq"].interface_name == "rabbitmq": + self._rabbitmq = RabbitMQRequires( self, - "amqp", + "rabbitmq", username=self.app.name, vhost="/", ) - self.framework.observe(self._amqp.on.connected, self._on_amqp_connected) - self.framework.observe(self._amqp.on.ready, self._on_amqp_ready) - self.framework.observe(self._amqp.on.goneaway, self._on_amqp_goneaway) + self.framework.observe(self._rabbitmq.on.connected, self._on_rabbitmq_connected) + self.framework.observe(self._rabbitmq.on.ready, self._on_rabbitmq_ready) + self.framework.observe(self._rabbitmq.on.goneaway, self._on_rabbitmq_goneaway) else: - self._amqp = None + self._rabbitmq = None self._database_migration = DatabaseMigration( container=self.unit.get_container(self._workload_config.container_name), @@ -288,9 +284,9 @@ def _missing_required_integrations(self, charm_state: CharmState) -> list[str]: if self._saml and not charm_state.integrations.saml_parameters: if not requires["saml"].optional: missing_integrations.append("saml") - if self._amqp and not charm_state.integrations.rabbitmq_parameters: - if not requires["amqp"].optional: - missing_integrations.append("amqp") + if self._rabbitmq and not charm_state.integrations.rabbitmq_parameters: + if not requires["rabbitmq"].optional: + missing_integrations.append("rabbitmq") return missing_integrations def restart(self) -> None: @@ -338,7 +334,7 @@ def _create_charm_state(self) -> CharmState: redis_uri=self._redis.url if self._redis is not None else None, s3_connection_info=self._s3.get_s3_connection_info() if self._s3 else None, saml_relation_data=saml_relation_data, - rabbitmq_parameters=self._amqp.rabbitmq_parameters() if self._amqp else None, + rabbitmq_parameters=self._rabbitmq.rabbitmq_parameters() if self._rabbitmq else None, base_url=self._base_url, ) @@ -440,16 +436,16 @@ def _on_pebble_ready(self, _: ops.PebbleReadyEvent) -> None: self.restart() @block_if_invalid_config - def _on_amqp_connected(self, _event: ops.HookEvent) -> None: + def _on_rabbitmq_connected(self, _event: ops.HookEvent) -> None: """Handle ampq connected event.""" self.restart() @block_if_invalid_config - def _on_amqp_ready(self, _event: ops.HookEvent) -> None: + def _on_rabbitmq_ready(self, _event: ops.HookEvent) -> None: """Handle ampq ready event.""" self.restart() @block_if_invalid_config - def _on_amqp_goneaway(self, _event: ops.HookEvent) -> None: + def _on_rabbitmq_goneaway(self, _event: ops.HookEvent) -> None: """Handle ampq goneaway event.""" self.restart() diff --git a/paas_app_charmer/rabbitmq.py b/paas_app_charmer/rabbitmq.py index 666083d..f3128cc 100644 --- a/paas_app_charmer/rabbitmq.py +++ b/paas_app_charmer/rabbitmq.py @@ -91,37 +91,37 @@ def __init__(self, charm: CharmBase, relation_name: str, username: str, vhost: s self.vhost = vhost self.framework.observe( self.charm.on[relation_name].relation_joined, - self._on_amqp_relation_joined, + self._on_rabbitmq_relation_joined, ) self.framework.observe( self.charm.on[relation_name].relation_changed, - self._on_amqp_relation_changed, + self._on_rabbitmq_relation_changed, ) self.framework.observe( self.charm.on[relation_name].relation_departed, - self._on_amqp_relation_changed, + self._on_rabbitmq_relation_changed, ) self.framework.observe( self.charm.on[relation_name].relation_broken, - self._on_amqp_relation_broken, + self._on_rabbitmq_relation_broken, ) - def _on_amqp_relation_joined(self, _: HookEvent) -> None: + def _on_rabbitmq_relation_joined(self, _: HookEvent) -> None: """Handle RabbitMQ joined.""" self.on.connected.emit() self.request_access(self.username, self.vhost) - def _on_amqp_relation_changed(self, _: HookEvent) -> None: + def _on_rabbitmq_relation_changed(self, _: HookEvent) -> None: """Handle RabbitMQ changed.""" if self.rabbitmq_parameters(): self.on.ready.emit() - def _on_amqp_relation_broken(self, _: HookEvent) -> None: + def _on_rabbitmq_relation_broken(self, _: HookEvent) -> None: """Handle RabbitMQ broken.""" self.on.goneaway.emit() @property - def _amqp_rel(self) -> Relation | None: + def _rabbitmq_rel(self) -> Relation | None: """The RabbitMQ relation.""" return self.framework.model.get_relation(self.relation_name) @@ -149,9 +149,9 @@ def request_access(self, username: str, vhost: str) -> None: vhost: virtual host requested for RabbitMQ """ if self.model.unit.is_leader(): - if self._amqp_rel: - self._amqp_rel.data[self.charm.app]["username"] = username - self._amqp_rel.data[self.charm.app]["vhost"] = vhost + if self._rabbitmq_rel: + self._rabbitmq_rel.data[self.charm.app]["username"] = username + self._rabbitmq_rel.data[self.charm.app]["vhost"] = vhost else: logger.warning("request_access but no rabbitmq relation") @@ -161,13 +161,13 @@ def _rabbitmq_server_parameters(self) -> RabbitMQParameters | None: Returns: Returns parameters for rabbitmq-server or None if they are not valid/complete. """ - if not self._amqp_rel: + if not self._rabbitmq_rel: return None password = None hostnames = [] - for unit in self._amqp_rel.units: - unit_data = self._amqp_rel.data[unit] + for unit in self._rabbitmq_rel.units: + unit_data = self._rabbitmq_rel.data[unit] # All of the passwords should be equal. If it is # in the unit data, get it and override the password password = unit_data.get("password", password) @@ -193,16 +193,16 @@ def _rabbitmq_k8s_parameters(self) -> RabbitMQParameters | None: Returns: Returns parameters for rabbitmq-k8s or None if they are not valid/complete. """ - if not self._amqp_rel: + if not self._rabbitmq_rel: return None - # A password in the _amqp_rel data differentiates rabbitmq-k8s from rabbitmq-server - password = self._amqp_rel.data[self._amqp_rel.app].get("password") - hostname = self._amqp_rel.data[self._amqp_rel.app].get("hostname") + # A password in the _rabbitmq_rel data differentiates rabbitmq-k8s from rabbitmq-server + password = self._rabbitmq_rel.data[self._rabbitmq_rel.app].get("password") + hostname = self._rabbitmq_rel.data[self._rabbitmq_rel.app].get("hostname") hostnames = [] - for unit in self._amqp_rel.units: - unit_data = self._amqp_rel.data[unit] + for unit in self._rabbitmq_rel.units: + unit_data = self._rabbitmq_rel.data[unit] ingress_address = unit_data.get("ingress-address") if ingress_address: hostnames.append(ingress_address) diff --git a/tests/integration/flask/conftest.py b/tests/integration/flask/conftest.py index e8a838b..d5dc9b8 100644 --- a/tests/integration/flask/conftest.py +++ b/tests/integration/flask/conftest.py @@ -275,7 +275,7 @@ async def rabbitmq_server_integration_fixture( yield integration - res = await flask_app.destroy_relation("amqp", f"{rabbitmq_server_app.name}:amqp") + res = await flask_app.destroy_relation("rabbitmq", f"{rabbitmq_server_app.name}:amqp") await model.wait_for_idle(apps=[flask_app.name], status="active") @@ -291,5 +291,5 @@ async def rabbitmq_k8s_integration_fixture( yield integration - await flask_app.destroy_relation("amqp", f"{rabbitmq_k8s_app.name}:amqp") + await flask_app.destroy_relation("rabbitmq", f"{rabbitmq_k8s_app.name}:amqp") await model.wait_for_idle(apps=[flask_app.name], status="active") diff --git a/tests/unit/flask/test_charm.py b/tests/unit/flask/test_charm.py index 9aa64e5..974b31d 100644 --- a/tests/unit/flask/test_charm.py +++ b/tests/unit/flask/test_charm.py @@ -165,7 +165,7 @@ def test_integrations_wiring(harness: Harness): @pytest.mark.parametrize( - "amqp_relation_data,expected_env_vars", + "rabbitmq_relation_data,expected_env_vars", [ pytest.param( { @@ -206,14 +206,14 @@ def test_integrations_wiring(harness: Harness): ), ], ) -def test_rabbitmq_integration(harness: Harness, amqp_relation_data, expected_env_vars): +def test_rabbitmq_integration(harness: Harness, rabbitmq_relation_data, expected_env_vars): """ - arrange: Prepare a amqp integration (RabbitMQ) + arrange: Prepare a rabbitmq integration (RabbitMQ) act: Start the flask charm and set flask-app container to be ready. assert: The flask service should have environment variables in its plan for each of the integrations. """ - harness.add_relation("amqp", "amqp", **amqp_relation_data) + harness.add_relation("rabbitmq", "rabbitmq", **rabbitmq_relation_data) container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) container.add_layer("a_layer", DEFAULT_LAYER) @@ -228,11 +228,11 @@ def test_rabbitmq_integration(harness: Harness, amqp_relation_data, expected_env def test_rabbitmq_integration_with_relation_data_empty(harness: Harness): """ - arrange: Prepare a amqp integration (RabbitMQ), with missing data. + arrange: Prepare a rabbitmq integration (RabbitMQ), with missing data. act: Start the flask charm and set flask-app container to be ready. assert: The flask service should not have environment variables related to RabbitMQ """ - harness.add_relation("amqp", "amqp") + harness.add_relation("rabbitmq", "rabbitmq") container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) container.add_layer("a_layer", DEFAULT_LAYER) @@ -246,12 +246,12 @@ def test_rabbitmq_integration_with_relation_data_empty(harness: Harness): def test_rabbitmq_remove_integration(harness: Harness): """ - arrange: Prepare a charm with a complete amqp integration (RabbitMQ). + arrange: Prepare a charm with a complete rabbitmq integration (RabbitMQ). act: Remove the relation. assert: The relation should not have the env variables related to RabbitMQ. """ relation_id = harness.add_relation( - "amqp", "amqp", app_data={"hostname": "example.com", "password": "p"} + "rabbitmq", "rabbitmq", app_data={"hostname": "example.com", "password": "p"} ) container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) container.add_layer("a_layer", DEFAULT_LAYER) @@ -272,7 +272,9 @@ def test_rabbitmq_remove_integration(harness: Harness): pytest.param(["saml"], ["s3"], id="s3 fails"), pytest.param(["redis", "s3"], ["mysql", "postgresql"], id="postgresql and mysql fail"), pytest.param( - [], ["mysql", "postgresql", "mongodb", "s3", "redis", "saml", "amqp"], id="all fail" + [], + ["mysql", "postgresql", "mongodb", "s3", "redis", "saml", "rabbitmq"], + id="all fail", ), ], ) From 41f553e8616db8ee2100f98fb9f2975bd97d62d0 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Thu, 5 Sep 2024 16:55:02 +0200 Subject: [PATCH 22/40] remove extra fetch-libs --- paas_app_charmer/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/paas_app_charmer/__init__.py b/paas_app_charmer/__init__.py index 03d0797..048d10c 100644 --- a/paas_app_charmer/__init__.py +++ b/paas_app_charmer/__init__.py @@ -19,35 +19,30 @@ except ImportError as import_error: raise exceptions.MissingCharmLibraryError( "Missing charm library, please run `charmcraft fetch-libs`" - "`charmcraft fetch-libs`" ) from import_error try: import charms.grafana_k8s.v0.grafana_dashboard # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( "Missing charm library, please run `charmcraft fetch-libs`" - "`charmcraft fetch-libs`" ) from import_error try: import charms.loki_k8s.v0.loki_push_api # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( "Missing charm library, please run `charmcraft fetch-libs`" - "`charmcraft fetch-libs`" ) from import_error try: import charms.prometheus_k8s.v0.prometheus_scrape # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( "Missing charm library, please run `charmcraft fetch-libs`" - "`charmcraft fetch-libs`" ) from import_error try: import charms.data_platform_libs.v0.data_interfaces # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( "Missing charm library, please run `charmcraft fetch-libs`" - "`charmcraft fetch-libs`" ) from import_error try: import charms.redis_k8s.v0.redis # noqa: F401 From df0dd9112b7a561a9b64d0764c6bfee514103f6e Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Thu, 5 Sep 2024 16:57:14 +0200 Subject: [PATCH 23/40] improve variables in tests --- tests/unit/flask/test_charm.py | 2 +- tests/unit/go/test_app.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/flask/test_charm.py b/tests/unit/flask/test_charm.py index 974b31d..767744c 100644 --- a/tests/unit/flask/test_charm.py +++ b/tests/unit/flask/test_charm.py @@ -251,7 +251,7 @@ def test_rabbitmq_remove_integration(harness: Harness): assert: The relation should not have the env variables related to RabbitMQ. """ relation_id = harness.add_relation( - "rabbitmq", "rabbitmq", app_data={"hostname": "example.com", "password": "p"} + "rabbitmq", "rabbitmq", app_data={"hostname": "example.com", "password": token_hex(16)} ) container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) container.add_layer("a_layer", DEFAULT_LAYER) diff --git a/tests/unit/go/test_app.py b/tests/unit/go/test_app.py index bd0a4fd..ba956bc 100644 --- a/tests/unit/go/test_app.py +++ b/tests/unit/go/test_app.py @@ -38,7 +38,7 @@ hostname="rabbitmq.example.com", hostnames=["127.0.0.1"], username="go-app", - password="nothingspecial", + password="test-password", vhost="./", ), ), @@ -61,7 +61,7 @@ "APP_REDIS_DB_QUERY": "", "APP_REDIS_DB_SCHEME": "redis", "APP_RABBITMQ_HOSTNAME": "rabbitmq.example.com", - "APP_RABBITMQ_PASSWORD": "nothingspecial", + "APP_RABBITMQ_PASSWORD": "test-password", "APP_RABBITMQ_USERNAME": "go-app", "APP_RABBITMQ_VHOST": "./", }, From 4a3d71cd754c038e043701f728f553f1b81abf07 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Fri, 6 Sep 2024 09:29:43 +0200 Subject: [PATCH 24/40] rename goneaway with departed --- paas_app_charmer/charm.py | 10 +++++----- paas_app_charmer/rabbitmq.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/paas_app_charmer/charm.py b/paas_app_charmer/charm.py index 8f0f13c..59b4e3c 100644 --- a/paas_app_charmer/charm.py +++ b/paas_app_charmer/charm.py @@ -108,7 +108,7 @@ def __init__(self, framework: ops.Framework, framework_name: str) -> None: ) self.framework.observe(self._rabbitmq.on.connected, self._on_rabbitmq_connected) self.framework.observe(self._rabbitmq.on.ready, self._on_rabbitmq_ready) - self.framework.observe(self._rabbitmq.on.goneaway, self._on_rabbitmq_goneaway) + self.framework.observe(self._rabbitmq.on.departed, self._on_rabbitmq_departed) else: self._rabbitmq = None @@ -437,15 +437,15 @@ def _on_pebble_ready(self, _: ops.PebbleReadyEvent) -> None: @block_if_invalid_config def _on_rabbitmq_connected(self, _event: ops.HookEvent) -> None: - """Handle ampq connected event.""" + """Handle rabbitmq connected event.""" self.restart() @block_if_invalid_config def _on_rabbitmq_ready(self, _event: ops.HookEvent) -> None: - """Handle ampq ready event.""" + """Handle rabbitmq ready event.""" self.restart() @block_if_invalid_config - def _on_rabbitmq_goneaway(self, _event: ops.HookEvent) -> None: - """Handle ampq goneaway event.""" + def _on_rabbitmq_departed(self, _event: ops.HookEvent) -> None: + """Handle rabbitmq departed event.""" self.restart() diff --git a/paas_app_charmer/rabbitmq.py b/paas_app_charmer/rabbitmq.py index f3128cc..bd5c18b 100644 --- a/paas_app_charmer/rabbitmq.py +++ b/paas_app_charmer/rabbitmq.py @@ -48,8 +48,8 @@ class RabbitMQReadyEvent(EventBase): """RabbitMQ ready for use Event.""" -class RabbitMQGoneAwayEvent(EventBase): - """RabbitMQ relation has gone-away Event.""" +class RabbitMQDepartedEvent(EventBase): + """RabbitMQ relation departed Event.""" class RabbitMQServerEvents(ObjectEvents): @@ -58,12 +58,12 @@ class RabbitMQServerEvents(ObjectEvents): Attributes: connected: rabbitmq relation is connected ready: rabbitmq relation is ready - goneaway: rabbitmq relation has been removed + departed: rabbitmq relation has been removed """ connected = EventSource(RabbitMQConnectedEvent) ready = EventSource(RabbitMQReadyEvent) - goneaway = EventSource(RabbitMQGoneAwayEvent) + departed = EventSource(RabbitMQDepartedEvent) class RabbitMQRequires(Object): @@ -118,7 +118,7 @@ def _on_rabbitmq_relation_changed(self, _: HookEvent) -> None: def _on_rabbitmq_relation_broken(self, _: HookEvent) -> None: """Handle RabbitMQ broken.""" - self.on.goneaway.emit() + self.on.departed.emit() @property def _rabbitmq_rel(self) -> Relation | None: From 7f99365872b7379f3d6543693d310fb3411c1a18 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Fri, 6 Sep 2024 11:00:27 +0200 Subject: [PATCH 25/40] Use amqp uri instead of our own struct --- examples/flask/test_rock/app.py | 3 +- paas_app_charmer/app.py | 90 +++++++++++++++++++++------------ paas_app_charmer/charm.py | 4 +- paas_app_charmer/charm_state.py | 34 +++---------- paas_app_charmer/rabbitmq.py | 65 ++++++++++++------------ tests/unit/flask/test_charm.py | 2 + tests/unit/go/test_app.py | 19 +++---- 7 files changed, 114 insertions(+), 103 deletions(-) diff --git a/examples/flask/test_rock/app.py b/examples/flask/test_rock/app.py index a928fe9..8a3a0c7 100644 --- a/examples/flask/test_rock/app.py +++ b/examples/flask/test_rock/app.py @@ -82,8 +82,9 @@ def get_rabbitmq_connection(): password = os.environ["RABBITMQ_PASSWORD"] hostname = os.environ["RABBITMQ_HOSTNAME"] vhost = os.environ["RABBITMQ_VHOST"] + port = os.environ["RABBITMQ_PORT"] credentials = pika.PlainCredentials(username, password) - parameters = pika.ConnectionParameters(hostname, 5672, vhost, credentials) + parameters = pika.ConnectionParameters(hostname, port, vhost, credentials) g.rabbitmq = pika.BlockingConnection(parameters) else: return None diff --git a/paas_app_charmer/app.py b/paas_app_charmer/app.py index 41010ba..3cd2ce3 100644 --- a/paas_app_charmer/app.py +++ b/paas_app_charmer/app.py @@ -219,10 +219,10 @@ def map_integrations_to_env(integrations: IntegrationsState, prefix: str = "") - """ env = {} if integrations.redis_uri: - redis_envvars = _db_url_to_env_variables("redis", integrations.redis_uri) + redis_envvars = _db_url_to_env_variables("REDIS", integrations.redis_uri) env.update(redis_envvars) for interface_name, uri in integrations.databases_uris.items(): - interface_envvars = _db_url_to_env_variables(interface_name, uri) + interface_envvars = _db_url_to_env_variables(interface_name.upper(), uri) env.update(interface_envvars) if integrations.s3_parameters: @@ -259,25 +259,18 @@ def map_integrations_to_env(integrations: IntegrationsState, prefix: str = "") - if v is not None ) - if integrations.rabbitmq_parameters: - rabbitmq = integrations.rabbitmq_parameters - env.update( - ( - ("RABBITMQ_HOSTNAME", rabbitmq.hostname), - ("RABBITMQ_USERNAME", rabbitmq.username), - ("RABBITMQ_PASSWORD", rabbitmq.password), - ("RABBITMQ_VHOST", rabbitmq.vhost), - ) - ) + if integrations.rabbitmq_uri: + rabbitmq_envvars = _rabbitmq_uri_to_env_variables("RABBITMQ", integrations.rabbitmq_uri) + env.update(rabbitmq_envvars) return {prefix + k: v for k, v in env.items()} -def _db_url_to_env_variables(base_name: str, url: str) -> dict[str, str]: +def _db_url_to_env_variables(prefix: str, url: str) -> dict[str, str]: """Convert a database url to environment variables. Args: - base_name: name of the database. + prefix: prefix for the environment variables url: url of the database Return: @@ -285,31 +278,66 @@ def _db_url_to_env_variables(base_name: str, url: str) -> dict[str, str]: all components as returned from urllib.parse and the database name extracted from the path """ + prefix = prefix + "_DB" + envvars = _url_env_vars(prefix, url) + parsed_url = urllib.parse.urlparse(url) + + # database name is usually parsed this way. + db_name = parsed_url.path.removeprefix("/") if parsed_url.path else None + if db_name is not None: + envvars[f"{prefix}_NAME"] = db_name + return envvars + + +def _rabbitmq_uri_to_env_variables(prefix: str, url: str) -> dict[str, str]: + """Convert a rabbitmq uri to environment variables. + + Args: + prefix: prefix for the environment variables + url: url of rabbitmq + + Return: + All environment variables, that is, the connection string, + all components as returned from urllib.parse and the + rabbitmq vhost extracted from the path + """ + envvars = _url_env_vars(prefix, url) + parsed_url = urllib.parse.urlparse(url) + if len(parsed_url.path) > 1: + envvars[f"{prefix}_VHOST"] = urllib.parse.unquote(parsed_url.path.split("/")[1]) + return envvars + + +def _url_env_vars(prefix: str, url: str) -> dict[str, str]: + """Convert a url to environment variables using parts from urllib.parse.urlparse. + + Args: + prefix: prefix for the environment variables + url: url of the database + + Return: + All environment variables, that is, the connection string and + all components as returned from urllib.parse + """ if not url: return {} - base_name = base_name.upper() envvars: dict[str, str | None] = {} - envvars[f"{base_name}_DB_CONNECT_STRING"] = url + envvars[f"{prefix}_CONNECT_STRING"] = url parsed_url = urllib.parse.urlparse(url) # All components of urlparse, using the same convention for default values. # See: https://docs.python.org/3/library/urllib.parse.html#url-parsing - envvars[f"{base_name}_DB_SCHEME"] = parsed_url.scheme - envvars[f"{base_name}_DB_NETLOC"] = parsed_url.netloc - envvars[f"{base_name}_DB_PATH"] = parsed_url.path - envvars[f"{base_name}_DB_PARAMS"] = parsed_url.params - envvars[f"{base_name}_DB_QUERY"] = parsed_url.query - envvars[f"{base_name}_DB_FRAGMENT"] = parsed_url.fragment - envvars[f"{base_name}_DB_USERNAME"] = parsed_url.username - envvars[f"{base_name}_DB_PASSWORD"] = parsed_url.password - envvars[f"{base_name}_DB_HOSTNAME"] = parsed_url.hostname - envvars[f"{base_name}_DB_PORT"] = str(parsed_url.port) if parsed_url.port is not None else None - - # database name is usually parsed this way. - envvars[f"{base_name}_DB_NAME"] = ( - parsed_url.path.removeprefix("/") if parsed_url.path else None - ) + envvars[f"{prefix}_SCHEME"] = parsed_url.scheme + envvars[f"{prefix}_NETLOC"] = parsed_url.netloc + envvars[f"{prefix}_PATH"] = parsed_url.path + envvars[f"{prefix}_PARAMS"] = parsed_url.params + envvars[f"{prefix}_QUERY"] = parsed_url.query + envvars[f"{prefix}_FRAGMENT"] = parsed_url.fragment + envvars[f"{prefix}_USERNAME"] = parsed_url.username + envvars[f"{prefix}_PASSWORD"] = parsed_url.password + envvars[f"{prefix}_HOSTNAME"] = parsed_url.hostname + envvars[f"{prefix}_PORT"] = str(parsed_url.port) if parsed_url.port is not None else None return {k: v for k, v in envvars.items() if v is not None} diff --git a/paas_app_charmer/charm.py b/paas_app_charmer/charm.py index 59b4e3c..723d0d7 100644 --- a/paas_app_charmer/charm.py +++ b/paas_app_charmer/charm.py @@ -284,7 +284,7 @@ def _missing_required_integrations(self, charm_state: CharmState) -> list[str]: if self._saml and not charm_state.integrations.saml_parameters: if not requires["saml"].optional: missing_integrations.append("saml") - if self._rabbitmq and not charm_state.integrations.rabbitmq_parameters: + if self._rabbitmq and not charm_state.integrations.rabbitmq_uri: if not requires["rabbitmq"].optional: missing_integrations.append("rabbitmq") return missing_integrations @@ -334,7 +334,7 @@ def _create_charm_state(self) -> CharmState: redis_uri=self._redis.url if self._redis is not None else None, s3_connection_info=self._s3.get_s3_connection_info() if self._s3 else None, saml_relation_data=saml_relation_data, - rabbitmq_parameters=self._rabbitmq.rabbitmq_parameters() if self._rabbitmq else None, + rabbitmq_uri=self._rabbitmq.rabbitmq_uri() if self._rabbitmq else None, base_url=self._base_url, ) diff --git a/paas_app_charmer/charm_state.py b/paas_app_charmer/charm_state.py index db01c0c..8cff8c5 100644 --- a/paas_app_charmer/charm_state.py +++ b/paas_app_charmer/charm_state.py @@ -88,7 +88,7 @@ def from_charm( # pylint: disable=too-many-arguments redis_uri: str | None = None, s3_connection_info: dict[str, str] | None = None, saml_relation_data: typing.MutableMapping[str, str] | None = None, - rabbitmq_parameters: "RabbitMQParameters | None" = None, + rabbitmq_uri: str | None = None, base_url: str | None = None, ) -> "CharmState": """Initialize a new instance of the CharmState class from the associated charm. @@ -102,7 +102,7 @@ def from_charm( # pylint: disable=too-many-arguments redis_uri: The redis uri provided by the redis charm. s3_connection_info: Connection info from S3 lib. saml_relation_data: Relation data from the SAML app. - rabbitmq_parameters: RabbitMQ parameters. + rabbitmq_uri: RabbitMQ uri. base_url: Base URL for the service. Return: @@ -122,7 +122,7 @@ def from_charm( # pylint: disable=too-many-arguments database_requirers=database_requirers, s3_connection_info=s3_connection_info, saml_relation_data=saml_relation_data, - rabbitmq_parameters=rabbitmq_parameters, + rabbitmq_uri=rabbitmq_uri, ) return cls( framework=framework, @@ -208,14 +208,14 @@ class IntegrationsState: databases_uris: Map from interface_name to the database uri. s3_parameters: S3 parameters. saml_parameters: SAML parameters. - rabbitmq_parameters: RabbitMQ parameters. + rabbitmq_uri: RabbitMQ uri. """ redis_uri: str | None = None databases_uris: dict[str, str] = field(default_factory=dict) s3_parameters: "S3Parameters | None" = None saml_parameters: "SamlParameters | None" = None - rabbitmq_parameters: "RabbitMQParameters | None" = None + rabbitmq_uri: str | None = None @classmethod def build( # pylint: disable=too-many-arguments @@ -224,7 +224,7 @@ def build( # pylint: disable=too-many-arguments database_requirers: dict[str, DatabaseRequires], s3_connection_info: dict[str, str] | None, saml_relation_data: typing.MutableMapping[str, str] | None = None, - rabbitmq_parameters: "RabbitMQParameters | None" = None, + rabbitmq_uri: str | None = None, ) -> "IntegrationsState": """Initialize a new instance of the IntegrationsState class. @@ -235,7 +235,7 @@ def build( # pylint: disable=too-many-arguments database_requirers: All database requirers object declared by the charm. s3_connection_info: S3 connection info from S3 lib. saml_relation_data: Saml relation data from saml lib. - rabbitmq_parameters: RabbitMQ parameters. + rabbitmq_uri: RabbitMQ uri. Return: The IntegrationsState instance created. @@ -283,7 +283,7 @@ def build( # pylint: disable=too-many-arguments }, s3_parameters=s3_parameters, saml_parameters=saml_parameters, - rabbitmq_parameters=rabbitmq_parameters, + rabbitmq_uri=rabbitmq_uri, ) @@ -362,21 +362,3 @@ def validate_signing_certificate_exists(cls, certs: str, _: ValidationInfo) -> s if not certificate: raise ValueError("Missing x509certs. There should be at least one certificate.") return certificate - - -class RabbitMQParameters(BaseModel, extra=Extra.allow): - """Configuration for accessing RabbitMQ. - - Attributes: - hostname: Hostname or ip address of broker. It could be one of the hostnames. - hostnames: List of Hostnames or ip addresses of broker. - username: The username to authenticate with - password: The password to authenticate with - vhost: RabbitMQ virtual host name. - """ - - hostname: str - hostnames: list[str] - username: str - password: str - vhost: str diff --git a/paas_app_charmer/rabbitmq.py b/paas_app_charmer/rabbitmq.py index bd5c18b..d22c5b4 100644 --- a/paas_app_charmer/rabbitmq.py +++ b/paas_app_charmer/rabbitmq.py @@ -30,13 +30,12 @@ import logging +import urllib.parse from ops import CharmBase, HookEvent from ops.framework import EventBase, EventSource, Object, ObjectEvents from ops.model import Relation -from paas_app_charmer.charm_state import RabbitMQParameters - logger = logging.getLogger(__name__) @@ -71,9 +70,11 @@ class RabbitMQRequires(Object): Attributes: on: ObjectEvents for RabbitMQRequires + port: amqp port """ on = RabbitMQServerEvents() + port = 5672 def __init__(self, charm: CharmBase, relation_name: str, username: str, vhost: str): """Initialize the instance. @@ -113,7 +114,7 @@ def _on_rabbitmq_relation_joined(self, _: HookEvent) -> None: def _on_rabbitmq_relation_changed(self, _: HookEvent) -> None: """Handle RabbitMQ changed.""" - if self.rabbitmq_parameters(): + if self.rabbitmq_uri(): self.on.ready.emit() def _on_rabbitmq_relation_broken(self, _: HookEvent) -> None: @@ -125,8 +126,8 @@ def _rabbitmq_rel(self) -> Relation | None: """The RabbitMQ relation.""" return self.framework.model.get_relation(self.relation_name) - def rabbitmq_parameters(self) -> RabbitMQParameters | None: - """Return RabbitMQ parameters with the data in the relation. + def rabbitmq_uri(self) -> str | None: + """Return RabbitMQ urs with the data in the relation. It will try to use the format in rabbitmq-k8s or rabbitmq-server. If there is no relation or the data is not complete, it returns None. @@ -134,12 +135,12 @@ def rabbitmq_parameters(self) -> RabbitMQParameters | None: Returns: The parameters for RabbitMQ or None. """ - rabbitmq_k8s_params = self._rabbitmq_k8s_parameters() + rabbitmq_k8s_params = self._rabbitmq_k8s_uri() if rabbitmq_k8s_params: return rabbitmq_k8s_params # rabbitmq-server parameters or None. - return self._rabbitmq_server_parameters() + return self._rabbitmq_server_uri() def request_access(self, username: str, vhost: str) -> None: """Request access to the RabbitMQ server. @@ -155,11 +156,11 @@ def request_access(self, username: str, vhost: str) -> None: else: logger.warning("request_access but no rabbitmq relation") - def _rabbitmq_server_parameters(self) -> RabbitMQParameters | None: - """Return parameters for rabbitmq-server. + def _rabbitmq_server_uri(self) -> str | None: + """Return uri for rabbitmq-server. Returns: - Returns parameters for rabbitmq-server or None if they are not valid/complete. + Returns uri for rabbitmq-server or None if the relation data is not valid/complete. """ if not self._rabbitmq_rel: return None @@ -179,19 +180,13 @@ def _rabbitmq_server_parameters(self) -> RabbitMQParameters | None: return None hostname = hostnames[0] - return RabbitMQParameters( - hostname=hostname, - hostnames=hostnames, - username=self.username, - password=password, - vhost=self.vhost, - ) + return self._build_amqp_uri(password=password, hostname=hostname) - def _rabbitmq_k8s_parameters(self) -> RabbitMQParameters | None: - """Return parameters for rabbitmq-k8s. + def _rabbitmq_k8s_uri(self) -> str | None: + """Return URI for rabbitmq-k8s. Returns: - Returns parameters for rabbitmq-k8s or None if they are not valid/complete. + Returns uri for rabbitmq-k8s or None if the relation data is not valid/complete. """ if not self._rabbitmq_rel: return None @@ -200,20 +195,22 @@ def _rabbitmq_k8s_parameters(self) -> RabbitMQParameters | None: password = self._rabbitmq_rel.data[self._rabbitmq_rel.app].get("password") hostname = self._rabbitmq_rel.data[self._rabbitmq_rel.app].get("hostname") - hostnames = [] - for unit in self._rabbitmq_rel.units: - unit_data = self._rabbitmq_rel.data[unit] - ingress_address = unit_data.get("ingress-address") - if ingress_address: - hostnames.append(ingress_address) - if not password or not hostname: return None - return RabbitMQParameters( - hostname=hostname, - hostnames=hostnames, - username=self.username, - password=password, - vhost=self.vhost, - ) + return self._build_amqp_uri(password=password, hostname=hostname) + + def _build_amqp_uri(self, password: str, hostname: str) -> str: + """Return amqp URI for rabbitmq from parameters. + + Args: + password: password for amqp uri + hostname: hostname for amqp uri + + Returns: + Returns amqp uri for rabbitmq from parameters + """ + # following https://www.rabbitmq.com/docs/uri-spec#the-amqp-uri-scheme, + # vhost component of a uri should be url encoded + vhost = urllib.parse.quote(self.vhost, safe="") + return f"amqp://{self.username}:{password}@{hostname}:{self.port}/{vhost}" diff --git a/tests/unit/flask/test_charm.py b/tests/unit/flask/test_charm.py index 767744c..ddcad7b 100644 --- a/tests/unit/flask/test_charm.py +++ b/tests/unit/flask/test_charm.py @@ -184,6 +184,7 @@ def test_integrations_wiring(harness: Harness): "RABBITMQ_USERNAME": "flask-k8s", "RABBITMQ_PASSWORD": "3m036hhyiDHs", "RABBITMQ_VHOST": "/", + "RABBITMQ_CONNECT_STRING": "amqp://flask-k8s:3m036hhyiDHs@rabbitmq-k8s-endpoints.testing.svc.cluster.local:5672/%2F", }, id="rabbitmq-k8s version", ), @@ -201,6 +202,7 @@ def test_integrations_wiring(harness: Harness): "RABBITMQ_USERNAME": "flask-k8s", "RABBITMQ_PASSWORD": "LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8", "RABBITMQ_VHOST": "/", + "RABBITMQ_CONNECT_STRING": "amqp://flask-k8s:LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8@10.58.171.158:5672/%2F", }, id="rabbitmq-server version", ), diff --git a/tests/unit/go/test_app.py b/tests/unit/go/test_app.py index ba956bc..4c2201b 100644 --- a/tests/unit/go/test_app.py +++ b/tests/unit/go/test_app.py @@ -9,7 +9,7 @@ import pytest from paas_app_charmer.app import App, WorkloadConfig -from paas_app_charmer.charm_state import CharmState, IntegrationsState, RabbitMQParameters +from paas_app_charmer.charm_state import CharmState, IntegrationsState from paas_app_charmer.go.charm import GoConfig @@ -34,13 +34,7 @@ {"metrics-port": "9000", "metrics-path": "/m", "app-secret-key": "notfoobar"}, IntegrationsState( redis_uri="redis://10.1.88.132:6379", - rabbitmq_parameters=RabbitMQParameters( - hostname="rabbitmq.example.com", - hostnames=["127.0.0.1"], - username="go-app", - password="test-password", - vhost="./", - ), + rabbitmq_uri="amqp://go-app:test-password@rabbitmq.example.com/%2f", ), { "APP_PORT": "8080", @@ -63,7 +57,14 @@ "APP_RABBITMQ_HOSTNAME": "rabbitmq.example.com", "APP_RABBITMQ_PASSWORD": "test-password", "APP_RABBITMQ_USERNAME": "go-app", - "APP_RABBITMQ_VHOST": "./", + "APP_RABBITMQ_VHOST": "/", + "APP_RABBITMQ_CONNECT_STRING": "amqp://go-app:test-password@rabbitmq.example.com/%2f", + "APP_RABBITMQ_FRAGMENT": "", + "APP_RABBITMQ_NETLOC": "go-app:test-password@rabbitmq.example.com", + "APP_RABBITMQ_PARAMS": "", + "APP_RABBITMQ_PATH": "/%2f", + "APP_RABBITMQ_QUERY": "", + "APP_RABBITMQ_SCHEME": "amqp", }, ), ], From 0a70624d8204bc3e21bb912465dda8505a8f7e9a Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Fri, 6 Sep 2024 11:23:16 +0200 Subject: [PATCH 26/40] Revert to previous comments for missing charm libraries --- paas_app_charmer/__init__.py | 19 ++++++++++++------- paas_app_charmer/charm.py | 12 ++++++++---- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/paas_app_charmer/__init__.py b/paas_app_charmer/__init__.py index 048d10c..570e36d 100644 --- a/paas_app_charmer/__init__.py +++ b/paas_app_charmer/__init__.py @@ -12,41 +12,46 @@ import charms.traefik_k8s.v2.ingress # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run `charmcraft fetch-libs`" + "Missing charm library, please run `charmcraft fetch-lib charms.traefik_k8s.v2.ingress`" ) from import_error try: import charms.observability_libs.v0.juju_topology # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run `charmcraft fetch-libs`" + "Missing charm library, please run " + "`charmcraft fetch-lib charms.observability_libs.v0.juju_topology`" ) from import_error try: import charms.grafana_k8s.v0.grafana_dashboard # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run `charmcraft fetch-libs`" + "Missing charm library, please run " + "`charmcraft fetch-lib charms.grafana_k8s.v0.grafana_dashboard`" ) from import_error try: import charms.loki_k8s.v0.loki_push_api # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run `charmcraft fetch-libs`" + "Missing charm library, please run " + "`charmcraft fetch-lib charms.loki_k8s.v0.loki_push_api`" ) from import_error try: import charms.prometheus_k8s.v0.prometheus_scrape # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run `charmcraft fetch-libs`" + "Missing charm library, please run " + "`charmcraft fetch-lib charms.prometheus_k8s.v0.prometheus_scrape`" ) from import_error try: import charms.data_platform_libs.v0.data_interfaces # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run `charmcraft fetch-libs`" + "Missing charm library, please run " + "`charmcraft fetch-lib charms.data_platform_libs.v0.data_interfaces`" ) from import_error try: import charms.redis_k8s.v0.redis # noqa: F401 except ImportError as import_error: raise exceptions.MissingCharmLibraryError( - "Missing charm library, please run `charmcraft fetch-libs`" + "Missing charm library, please run `charmcraft fetch-lib charms.redis_k8s.v0.redis`" ) from import_error diff --git a/paas_app_charmer/charm.py b/paas_app_charmer/charm.py index 723d0d7..0938682 100644 --- a/paas_app_charmer/charm.py +++ b/paas_app_charmer/charm.py @@ -25,19 +25,23 @@ logger = logging.getLogger(__name__) -# Do not fail for new integrations to be backwards compatible without forcing -# the user to run fetch-libs. +# Until charmcraft fetch-libs is implemented, the charm will not fail +# if new optional libs are not fetched, as it will not be backwards compatible. try: # pylint: disable=ungrouped-imports from charms.data_platform_libs.v0.s3 import S3Requirer except ImportError: - logger.exception("Missing charm library, please run `charmcraft fetch-libs`") + logger.exception( + "Missing charm library, please run `charmcraft fetch-lib charms.data_platform_libs.v0.s3`" + ) try: # pylint: disable=ungrouped-imports from charms.saml_integrator.v0.saml import SamlRequires except ImportError: - logger.exception("Missing charm library, please run `charmcraft fetch-libs`") + logger.exception( + "Missing charm library, please run `charmcraft fetch-lib charms.saml_integrator.v0.saml`" + ) class PaasCharm(abc.ABC, ops.CharmBase): # pylint: disable=too-many-instance-attributes From 9d266ce6f5d23ed8d687da5d96acfdf343697b36 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Fri, 6 Sep 2024 11:45:43 +0200 Subject: [PATCH 27/40] get rabbitmq connection from individual parts and frmo uri to check eveything is ok --- examples/flask/test_rock/app.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/examples/flask/test_rock/app.py b/examples/flask/test_rock/app.py index 8a3a0c7..6b9fe99 100644 --- a/examples/flask/test_rock/app.py +++ b/examples/flask/test_rock/app.py @@ -91,6 +91,18 @@ def get_rabbitmq_connection(): return g.rabbitmq +def get_rabbitmq_connection_from_uri(): + """Get rabbitmq connection from uri.""" + if "rabbitmq_from_uri" not in g: + if "RABBITMQ_CONNECT_STRING" in os.environ: + uri = os.environ["RABBITMQ_CONNECT_STRING"] + parameters = pika.URLParameters(uri) + g.rabbitmq_from_uri = pika.BlockingConnection(parameters) + else: + return None + return g.rabbitmq_from_uri + + def get_boto3_client(): if "boto3_client" not in g: if "S3_ACCESS_KEY" in os.environ: @@ -219,7 +231,7 @@ def rabbitmq_send(): @app.route("/rabbitmq/receive") def rabbitmq_receive(): """Receive a message from "charm" queue in blocking form.""" - if connection := get_rabbitmq_connection(): + if connection := get_rabbitmq_connection_from_uri(): channel = connection.channel() method_frame, _header_frame, body = channel.basic_get("charm") if method_frame: From dd27db8c38613d640cfdeb67f604abb7e74370cf Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Fri, 6 Sep 2024 12:00:29 +0200 Subject: [PATCH 28/40] add comment for lint exception --- paas_app_charmer/charm_state.py | 1 + 1 file changed, 1 insertion(+) diff --git a/paas_app_charmer/charm_state.py b/paas_app_charmer/charm_state.py index 8cff8c5..ca30baf 100644 --- a/paas_app_charmer/charm_state.py +++ b/paas_app_charmer/charm_state.py @@ -217,6 +217,7 @@ class IntegrationsState: saml_parameters: "SamlParameters | None" = None rabbitmq_uri: str | None = None + # This dataclass combines all the integrations, so it is reasonable that they stay together. @classmethod def build( # pylint: disable=too-many-arguments cls, From 0c2b00efae0eca0b278626eb2c1efde6b52f1736 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 10 Sep 2024 10:17:46 +0200 Subject: [PATCH 29/40] add trivy ignore --- .trivyignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.trivyignore b/.trivyignore index 2592ea4..ff1172d 100644 --- a/.trivyignore +++ b/.trivyignore @@ -6,3 +6,5 @@ CVE-2023-45288 CVE-2022-40897 # pypa/setuptools: Remote code execution via download CVE-2024-6345 +# pebble: Calling Decoder.Decode on a message which contains deeply nested structures can cause a panic due to stack exhaustion +CVE-2024-34156 From 1ea32b5f79dea9573b72d555bf4430e31498e386 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 10 Sep 2024 10:43:39 +0200 Subject: [PATCH 30/40] Launch actions From 18d98767bff610e7fff6360aa169f8248fa0745f Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 10 Sep 2024 12:36:27 +0200 Subject: [PATCH 31/40] test remove first trivy --- .trivyignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.trivyignore b/.trivyignore index ff1172d..fc6c6f8 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,5 +1,5 @@ -# stdlib golang: net/netip: Unexpected behavior from Is methods for IPv4-mapped IPv6 addresses -CVE-2024-24790 +# # stdlib golang: net/netip: Unexpected behavior from Is methods for IPv4-mapped IPv6 addresses +# CVE-2024-24790 # CVE usr/bin/pebble (gobinary) CVE-2023-45288 # ignore CVE introduced by python3-gunicorn From 9b1c752ba3936eaf4ce613c877ff74cd402e38e8 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 10 Sep 2024 12:56:05 +0200 Subject: [PATCH 32/40] remove CVE-2024-24790 and comment CVE-2023-45288 --- .trivyignore | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.trivyignore b/.trivyignore index fc6c6f8..bab19e7 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,7 +1,5 @@ -# # stdlib golang: net/netip: Unexpected behavior from Is methods for IPv4-mapped IPv6 addresses -# CVE-2024-24790 -# CVE usr/bin/pebble (gobinary) -CVE-2023-45288 +# # CVE usr/bin/pebble (gobinary) +# CVE-2023-45288 # ignore CVE introduced by python3-gunicorn CVE-2022-40897 # pypa/setuptools: Remote code execution via download From 45249eb2eda679dfd163dff1c0f3c72520cd9a12 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 10 Sep 2024 13:10:22 +0200 Subject: [PATCH 33/40] remove CVE-2023-45288. comment CVE-2022-40897 --- .trivyignore | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.trivyignore b/.trivyignore index bab19e7..a8f2d55 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,7 +1,5 @@ -# # CVE usr/bin/pebble (gobinary) -# CVE-2023-45288 -# ignore CVE introduced by python3-gunicorn -CVE-2022-40897 +# # ignore CVE introduced by python3-gunicorn +# CVE-2022-40897 # pypa/setuptools: Remote code execution via download CVE-2024-6345 # pebble: Calling Decoder.Decode on a message which contains deeply nested structures can cause a panic due to stack exhaustion From 90c453df3166d494c2c71bd6121bca077522cf2e Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 10 Sep 2024 13:24:02 +0200 Subject: [PATCH 34/40] lets see CVE-2024-6345 --- .trivyignore | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.trivyignore b/.trivyignore index a8f2d55..2186a08 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,6 +1,6 @@ -# # ignore CVE introduced by python3-gunicorn -# CVE-2022-40897 -# pypa/setuptools: Remote code execution via download -CVE-2024-6345 +# ignore CVE introduced by python3-gunicorn +CVE-2022-40897 +# # pypa/setuptools: Remote code execution via download +# CVE-2024-6345 # pebble: Calling Decoder.Decode on a message which contains deeply nested structures can cause a panic due to stack exhaustion CVE-2024-34156 From 6358f71397cc3779e44ccfe212c8f66dae524b15 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Tue, 10 Sep 2024 13:36:36 +0200 Subject: [PATCH 35/40] put the necessary ignores in trivvy --- .trivyignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.trivyignore b/.trivyignore index 2186a08..f39633c 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,6 +1,6 @@ # ignore CVE introduced by python3-gunicorn CVE-2022-40897 -# # pypa/setuptools: Remote code execution via download -# CVE-2024-6345 +# pypa/setuptools: Remote code execution via download +CVE-2024-6345 # pebble: Calling Decoder.Decode on a message which contains deeply nested structures can cause a panic due to stack exhaustion CVE-2024-34156 From bd1933fc31aa89c4b4927648f28235cb51d4dbba Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 11 Sep 2024 13:09:27 +0200 Subject: [PATCH 36/40] do not use self hosted runners temporarily --- .github/workflows/test.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bd1426c..ee5fa3e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,6 +7,3 @@ jobs: unit-tests: uses: canonical/operator-workflows/.github/workflows/test.yaml@main secrets: inherit - with: - self-hosted-runner: true - self-hosted-runner-label: "edge" From 8d8b04e1a085625303b5fd4570945f78c22f4239 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 11 Sep 2024 13:14:57 +0200 Subject: [PATCH 37/40] do not use self hosted runners --- .github/workflows/test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ee5fa3e..39029a1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,3 +7,6 @@ jobs: unit-tests: uses: canonical/operator-workflows/.github/workflows/test.yaml@main secrets: inherit + with: + self-hosted-runner: false + self-hosted-runner-label: "edge" From 172b8207a1f202374d99350777a7c48c6a817da1 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 11 Sep 2024 13:29:29 +0200 Subject: [PATCH 38/40] Add typing hints to get rabbitmq connections --- examples/flask/test_rock/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/flask/test_rock/app.py b/examples/flask/test_rock/app.py index 6b9fe99..2b6de2d 100644 --- a/examples/flask/test_rock/app.py +++ b/examples/flask/test_rock/app.py @@ -74,7 +74,7 @@ def get_redis_database() -> redis.Redis | None: return g.redis_db -def get_rabbitmq_connection(): +def get_rabbitmq_connection() -> pika.BlockingConnection | None: """Get rabbitmq connection.""" if "rabbitmq" not in g: if "RABBITMQ_HOSTNAME" in os.environ: @@ -91,7 +91,7 @@ def get_rabbitmq_connection(): return g.rabbitmq -def get_rabbitmq_connection_from_uri(): +def get_rabbitmq_connection_from_uri() -> pika.BlockingConnection | None: """Get rabbitmq connection from uri.""" if "rabbitmq_from_uri" not in g: if "RABBITMQ_CONNECT_STRING" in os.environ: From f266001f70fec581d2af5beae279200c19eb5a4c Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 11 Sep 2024 14:06:24 +0200 Subject: [PATCH 39/40] Code suggestions --- paas_app_charmer/rabbitmq.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/paas_app_charmer/rabbitmq.py b/paas_app_charmer/rabbitmq.py index d22c5b4..41ff4af 100644 --- a/paas_app_charmer/rabbitmq.py +++ b/paas_app_charmer/rabbitmq.py @@ -100,7 +100,7 @@ def __init__(self, charm: CharmBase, relation_name: str, username: str, vhost: s ) self.framework.observe( self.charm.on[relation_name].relation_departed, - self._on_rabbitmq_relation_changed, + self._on_rabbitmq_relation_departed, ) self.framework.observe( self.charm.on[relation_name].relation_broken, @@ -117,6 +117,11 @@ def _on_rabbitmq_relation_changed(self, _: HookEvent) -> None: if self.rabbitmq_uri(): self.on.ready.emit() + def _on_rabbitmq_relation_departed(self, _: HookEvent) -> None: + """Handle RabbitMQ departed.""" + if self.rabbitmq_uri(): + self.on.ready.emit() + def _on_rabbitmq_relation_broken(self, _: HookEvent) -> None: """Handle RabbitMQ broken.""" self.on.departed.emit() @@ -150,11 +155,11 @@ def request_access(self, username: str, vhost: str) -> None: vhost: virtual host requested for RabbitMQ """ if self.model.unit.is_leader(): - if self._rabbitmq_rel: - self._rabbitmq_rel.data[self.charm.app]["username"] = username - self._rabbitmq_rel.data[self.charm.app]["vhost"] = vhost - else: + if not self._rabbitmq_rel: logger.warning("request_access but no rabbitmq relation") + return + self._rabbitmq_rel.data[self.charm.app]["username"] = username + self._rabbitmq_rel.data[self.charm.app]["vhost"] = vhost def _rabbitmq_server_uri(self) -> str | None: """Return uri for rabbitmq-server. From 8fbcc597a8f18b8482cc7f10ff1667aa28f1bb89 Mon Sep 17 00:00:00 2001 From: Javier de la Puente Date: Wed, 11 Sep 2024 15:59:35 +0200 Subject: [PATCH 40/40] close rabbitmq connections on teardown --- examples/flask/test_rock/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/flask/test_rock/app.py b/examples/flask/test_rock/app.py index 2b6de2d..2f21b52 100644 --- a/examples/flask/test_rock/app.py +++ b/examples/flask/test_rock/app.py @@ -143,6 +143,12 @@ def teardown_database(_): boto3_client = g.pop("boto3_client", None) if boto3_client is not None: boto3_client.close() + rabbitmq = g.pop("rabbitmq", None) + if rabbitmq is not None: + rabbitmq.close() + rabbitmq_from_uri = g.pop("rabbitmq_from_uri", None) + if rabbitmq_from_uri is not None: + rabbitmq_from_uri.close() @app.route("/")