From 021ae380c04e0ddb1a5dffdda81c384edb353dee Mon Sep 17 00:00:00 2001 From: javierdelapuente Date: Wed, 11 Sep 2024 17:04:00 +0200 Subject: [PATCH] RabbitMQ integration (#39) --- .github/workflows/test.yaml | 2 +- .trivyignore | 6 +- examples/flask/charmcraft.yaml | 5 + examples/flask/test_rock/app.py | 62 ++++++ examples/flask/test_rock/requirements.txt | 1 + paas_app_charmer/app.py | 81 +++++-- paas_app_charmer/charm.py | 37 +++- paas_app_charmer/charm_state.py | 11 +- paas_app_charmer/rabbitmq.py | 221 +++++++++++++++++++ tests/integration/conftest.py | 58 ++++- tests/integration/flask/conftest.py | 41 ++++ tests/integration/flask/test_integrations.py | 40 ++++ tests/unit/flask/test_charm.py | 112 +++++++++- tests/unit/go/test_app.py | 16 +- 14 files changed, 661 insertions(+), 32 deletions(-) create mode 100644 paas_app_charmer/rabbitmq.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bd1426c..39029a1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,5 +8,5 @@ jobs: uses: canonical/operator-workflows/.github/workflows/test.yaml@main secrets: inherit with: - self-hosted-runner: true + self-hosted-runner: false self-hosted-runner-label: "edge" diff --git a/.trivyignore b/.trivyignore index 2592ea4..f39633c 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,8 +1,6 @@ -# stdlib golang: net/netip: Unexpected behavior from Is methods for IPv4-mapped IPv6 addresses -CVE-2024-24790 -# CVE usr/bin/pebble (gobinary) -CVE-2023-45288 # ignore CVE introduced by python3-gunicorn CVE-2022-40897 # pypa/setuptools: Remote code execution via download CVE-2024-6345 +# pebble: Calling Decoder.Decode on a message which contains deeply nested structures can cause a panic due to stack exhaustion +CVE-2024-34156 diff --git a/examples/flask/charmcraft.yaml b/examples/flask/charmcraft.yaml index 160f2d7..ca7894f 100644 --- a/examples/flask/charmcraft.yaml +++ b/examples/flask/charmcraft.yaml @@ -115,6 +115,11 @@ requires: interface: saml optional: True limit: 1 + rabbitmq: + interface: rabbitmq + optional: True + limit: 1 + resources: flask-app-image: description: flask application image. diff --git a/examples/flask/test_rock/app.py b/examples/flask/test_rock/app.py index dcefa1b..2f21b52 100644 --- a/examples/flask/test_rock/app.py +++ b/examples/flask/test_rock/app.py @@ -8,6 +8,7 @@ import boto3 import botocore.config +import pika import psycopg import pymongo import pymongo.database @@ -73,6 +74,35 @@ def get_redis_database() -> redis.Redis | None: return g.redis_db +def get_rabbitmq_connection() -> pika.BlockingConnection | None: + """Get rabbitmq connection.""" + if "rabbitmq" not in g: + if "RABBITMQ_HOSTNAME" in os.environ: + username = os.environ["RABBITMQ_USERNAME"] + password = os.environ["RABBITMQ_PASSWORD"] + hostname = os.environ["RABBITMQ_HOSTNAME"] + vhost = os.environ["RABBITMQ_VHOST"] + port = os.environ["RABBITMQ_PORT"] + credentials = pika.PlainCredentials(username, password) + parameters = pika.ConnectionParameters(hostname, port, vhost, credentials) + g.rabbitmq = pika.BlockingConnection(parameters) + else: + return None + return g.rabbitmq + + +def get_rabbitmq_connection_from_uri() -> pika.BlockingConnection | None: + """Get rabbitmq connection from uri.""" + if "rabbitmq_from_uri" not in g: + if "RABBITMQ_CONNECT_STRING" in os.environ: + uri = os.environ["RABBITMQ_CONNECT_STRING"] + parameters = pika.URLParameters(uri) + g.rabbitmq_from_uri = pika.BlockingConnection(parameters) + else: + return None + return g.rabbitmq_from_uri + + def get_boto3_client(): if "boto3_client" not in g: if "S3_ACCESS_KEY" in os.environ: @@ -113,6 +143,12 @@ def teardown_database(_): boto3_client = g.pop("boto3_client", None) if boto3_client is not None: boto3_client.close() + rabbitmq = g.pop("rabbitmq", None) + if rabbitmq is not None: + rabbitmq.close() + rabbitmq_from_uri = g.pop("rabbitmq_from_uri", None) + if rabbitmq_from_uri is not None: + rabbitmq_from_uri.close() @app.route("/") @@ -187,6 +223,32 @@ def redis_status(): return "FAIL" +@app.route("/rabbitmq/send") +def rabbitmq_send(): + """Send a message to "charm" queue.""" + if connection := get_rabbitmq_connection(): + channel = connection.channel() + channel.queue_declare(queue="charm") + channel.basic_publish(exchange="", routing_key="charm", body="SUCCESS") + return "SUCCESS" + return "FAIL" + + +@app.route("/rabbitmq/receive") +def rabbitmq_receive(): + """Receive a message from "charm" queue in blocking form.""" + if connection := get_rabbitmq_connection_from_uri(): + channel = connection.channel() + method_frame, _header_frame, body = channel.basic_get("charm") + if method_frame: + channel.basic_ack(method_frame.delivery_tag) + if body == b"SUCCESS": + return "SUCCESS" + return "FAIL. INCORRECT MESSAGE." + return "FAIL. NO MESSAGE." + return "FAIL. NO CONNECTION." + + @app.route("/env") def get_env(): """Return environment variables""" diff --git a/examples/flask/test_rock/requirements.txt b/examples/flask/test_rock/requirements.txt index 4c2df06..5f1ab38 100644 --- a/examples/flask/test_rock/requirements.txt +++ b/examples/flask/test_rock/requirements.txt @@ -6,3 +6,4 @@ psycopg[binary] pymongo redis[hiredis] boto3 +pika diff --git a/paas_app_charmer/app.py b/paas_app_charmer/app.py index 678e5db..3cd2ce3 100644 --- a/paas_app_charmer/app.py +++ b/paas_app_charmer/app.py @@ -219,10 +219,10 @@ def map_integrations_to_env(integrations: IntegrationsState, prefix: str = "") - """ env = {} if integrations.redis_uri: - redis_envvars = _db_url_to_env_variables("redis", integrations.redis_uri) + redis_envvars = _db_url_to_env_variables("REDIS", integrations.redis_uri) env.update(redis_envvars) for interface_name, uri in integrations.databases_uris.items(): - interface_envvars = _db_url_to_env_variables(interface_name, uri) + interface_envvars = _db_url_to_env_variables(interface_name.upper(), uri) env.update(interface_envvars) if integrations.s3_parameters: @@ -259,14 +259,18 @@ def map_integrations_to_env(integrations: IntegrationsState, prefix: str = "") - if v is not None ) + if integrations.rabbitmq_uri: + rabbitmq_envvars = _rabbitmq_uri_to_env_variables("RABBITMQ", integrations.rabbitmq_uri) + env.update(rabbitmq_envvars) + return {prefix + k: v for k, v in env.items()} -def _db_url_to_env_variables(base_name: str, url: str) -> dict[str, str]: +def _db_url_to_env_variables(prefix: str, url: str) -> dict[str, str]: """Convert a database url to environment variables. Args: - base_name: name of the database. + prefix: prefix for the environment variables url: url of the database Return: @@ -274,31 +278,66 @@ def _db_url_to_env_variables(base_name: str, url: str) -> dict[str, str]: all components as returned from urllib.parse and the database name extracted from the path """ + prefix = prefix + "_DB" + envvars = _url_env_vars(prefix, url) + parsed_url = urllib.parse.urlparse(url) + + # database name is usually parsed this way. + db_name = parsed_url.path.removeprefix("/") if parsed_url.path else None + if db_name is not None: + envvars[f"{prefix}_NAME"] = db_name + return envvars + + +def _rabbitmq_uri_to_env_variables(prefix: str, url: str) -> dict[str, str]: + """Convert a rabbitmq uri to environment variables. + + Args: + prefix: prefix for the environment variables + url: url of rabbitmq + + Return: + All environment variables, that is, the connection string, + all components as returned from urllib.parse and the + rabbitmq vhost extracted from the path + """ + envvars = _url_env_vars(prefix, url) + parsed_url = urllib.parse.urlparse(url) + if len(parsed_url.path) > 1: + envvars[f"{prefix}_VHOST"] = urllib.parse.unquote(parsed_url.path.split("/")[1]) + return envvars + + +def _url_env_vars(prefix: str, url: str) -> dict[str, str]: + """Convert a url to environment variables using parts from urllib.parse.urlparse. + + Args: + prefix: prefix for the environment variables + url: url of the database + + Return: + All environment variables, that is, the connection string and + all components as returned from urllib.parse + """ if not url: return {} - base_name = base_name.upper() envvars: dict[str, str | None] = {} - envvars[f"{base_name}_DB_CONNECT_STRING"] = url + envvars[f"{prefix}_CONNECT_STRING"] = url parsed_url = urllib.parse.urlparse(url) # All components of urlparse, using the same convention for default values. # See: https://docs.python.org/3/library/urllib.parse.html#url-parsing - envvars[f"{base_name}_DB_SCHEME"] = parsed_url.scheme - envvars[f"{base_name}_DB_NETLOC"] = parsed_url.netloc - envvars[f"{base_name}_DB_PATH"] = parsed_url.path - envvars[f"{base_name}_DB_PARAMS"] = parsed_url.params - envvars[f"{base_name}_DB_QUERY"] = parsed_url.query - envvars[f"{base_name}_DB_FRAGMENT"] = parsed_url.fragment - envvars[f"{base_name}_DB_USERNAME"] = parsed_url.username - envvars[f"{base_name}_DB_PASSWORD"] = parsed_url.password - envvars[f"{base_name}_DB_HOSTNAME"] = parsed_url.hostname - envvars[f"{base_name}_DB_PORT"] = str(parsed_url.port) if parsed_url.port is not None else None - - # database name is usually parsed this way. - envvars[f"{base_name}_DB_NAME"] = ( - parsed_url.path.removeprefix("/") if parsed_url.path else None - ) + envvars[f"{prefix}_SCHEME"] = parsed_url.scheme + envvars[f"{prefix}_NETLOC"] = parsed_url.netloc + envvars[f"{prefix}_PATH"] = parsed_url.path + envvars[f"{prefix}_PARAMS"] = parsed_url.params + envvars[f"{prefix}_QUERY"] = parsed_url.query + envvars[f"{prefix}_FRAGMENT"] = parsed_url.fragment + envvars[f"{prefix}_USERNAME"] = parsed_url.username + envvars[f"{prefix}_PASSWORD"] = parsed_url.password + envvars[f"{prefix}_HOSTNAME"] = parsed_url.hostname + envvars[f"{prefix}_PORT"] = str(parsed_url.port) if parsed_url.port is not None else None return {k: v for k, v in envvars.items() if v is not None} diff --git a/paas_app_charmer/charm.py b/paas_app_charmer/charm.py index 6f85720..0938682 100644 --- a/paas_app_charmer/charm.py +++ b/paas_app_charmer/charm.py @@ -19,6 +19,7 @@ from paas_app_charmer.databases import make_database_requirers from paas_app_charmer.exceptions import CharmConfigInvalidError from paas_app_charmer.observability import Observability +from paas_app_charmer.rabbitmq import RabbitMQRequires from paas_app_charmer.secret_storage import KeySecretStorage from paas_app_charmer.utils import build_validation_error_message @@ -101,6 +102,20 @@ def __init__(self, framework: ops.Framework, framework_name: str) -> None: else: self._saml = None + self._rabbitmq: RabbitMQRequires | None + if "rabbitmq" in requires and requires["rabbitmq"].interface_name == "rabbitmq": + self._rabbitmq = RabbitMQRequires( + self, + "rabbitmq", + username=self.app.name, + vhost="/", + ) + self.framework.observe(self._rabbitmq.on.connected, self._on_rabbitmq_connected) + self.framework.observe(self._rabbitmq.on.ready, self._on_rabbitmq_ready) + self.framework.observe(self._rabbitmq.on.departed, self._on_rabbitmq_departed) + else: + self._rabbitmq = None + self._database_migration = DatabaseMigration( container=self.unit.get_container(self._workload_config.container_name), state_dir=self._workload_config.state_dir, @@ -245,7 +260,8 @@ def is_ready(self) -> bool: return True - def _missing_required_integrations(self, charm_state: CharmState) -> list[str]: + # Pending to refactor all integrations + def _missing_required_integrations(self, charm_state: CharmState) -> list[str]: # noqa: C901 """Get list of missing integrations that are required. Args: @@ -272,6 +288,9 @@ def _missing_required_integrations(self, charm_state: CharmState) -> list[str]: if self._saml and not charm_state.integrations.saml_parameters: if not requires["saml"].optional: missing_integrations.append("saml") + if self._rabbitmq and not charm_state.integrations.rabbitmq_uri: + if not requires["rabbitmq"].optional: + missing_integrations.append("rabbitmq") return missing_integrations def restart(self) -> None: @@ -319,6 +338,7 @@ def _create_charm_state(self) -> CharmState: redis_uri=self._redis.url if self._redis is not None else None, s3_connection_info=self._s3.get_s3_connection_info() if self._s3 else None, saml_relation_data=saml_relation_data, + rabbitmq_uri=self._rabbitmq.rabbitmq_uri() if self._rabbitmq else None, base_url=self._base_url, ) @@ -418,3 +438,18 @@ def _on_ingress_ready(self, _: ops.HookEvent) -> None: def _on_pebble_ready(self, _: ops.PebbleReadyEvent) -> None: """Handle the pebble-ready event.""" self.restart() + + @block_if_invalid_config + def _on_rabbitmq_connected(self, _event: ops.HookEvent) -> None: + """Handle rabbitmq connected event.""" + self.restart() + + @block_if_invalid_config + def _on_rabbitmq_ready(self, _event: ops.HookEvent) -> None: + """Handle rabbitmq ready event.""" + self.restart() + + @block_if_invalid_config + def _on_rabbitmq_departed(self, _event: ops.HookEvent) -> None: + """Handle rabbitmq departed event.""" + self.restart() diff --git a/paas_app_charmer/charm_state.py b/paas_app_charmer/charm_state.py index ac920e8..ca30baf 100644 --- a/paas_app_charmer/charm_state.py +++ b/paas_app_charmer/charm_state.py @@ -88,6 +88,7 @@ def from_charm( # pylint: disable=too-many-arguments redis_uri: str | None = None, s3_connection_info: dict[str, str] | None = None, saml_relation_data: typing.MutableMapping[str, str] | None = None, + rabbitmq_uri: str | None = None, base_url: str | None = None, ) -> "CharmState": """Initialize a new instance of the CharmState class from the associated charm. @@ -101,6 +102,7 @@ def from_charm( # pylint: disable=too-many-arguments redis_uri: The redis uri provided by the redis charm. s3_connection_info: Connection info from S3 lib. saml_relation_data: Relation data from the SAML app. + rabbitmq_uri: RabbitMQ uri. base_url: Base URL for the service. Return: @@ -120,6 +122,7 @@ def from_charm( # pylint: disable=too-many-arguments database_requirers=database_requirers, s3_connection_info=s3_connection_info, saml_relation_data=saml_relation_data, + rabbitmq_uri=rabbitmq_uri, ) return cls( framework=framework, @@ -205,20 +208,24 @@ class IntegrationsState: databases_uris: Map from interface_name to the database uri. s3_parameters: S3 parameters. saml_parameters: SAML parameters. + rabbitmq_uri: RabbitMQ uri. """ redis_uri: str | None = None databases_uris: dict[str, str] = field(default_factory=dict) s3_parameters: "S3Parameters | None" = None saml_parameters: "SamlParameters | None" = None + rabbitmq_uri: str | None = None + # This dataclass combines all the integrations, so it is reasonable that they stay together. @classmethod - def build( + def build( # pylint: disable=too-many-arguments cls, redis_uri: str | None, database_requirers: dict[str, DatabaseRequires], s3_connection_info: dict[str, str] | None, saml_relation_data: typing.MutableMapping[str, str] | None = None, + rabbitmq_uri: str | None = None, ) -> "IntegrationsState": """Initialize a new instance of the IntegrationsState class. @@ -229,6 +236,7 @@ def build( database_requirers: All database requirers object declared by the charm. s3_connection_info: S3 connection info from S3 lib. saml_relation_data: Saml relation data from saml lib. + rabbitmq_uri: RabbitMQ uri. Return: The IntegrationsState instance created. @@ -276,6 +284,7 @@ def build( }, s3_parameters=s3_parameters, saml_parameters=saml_parameters, + rabbitmq_uri=rabbitmq_uri, ) diff --git a/paas_app_charmer/rabbitmq.py b/paas_app_charmer/rabbitmq.py new file mode 100644 index 0000000..41ff4af --- /dev/null +++ b/paas_app_charmer/rabbitmq.py @@ -0,0 +1,221 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""RabbitMQ library for handling the rabbitmq interface. + +The project https://github.com/openstack-charmers/charm-rabbitmq-k8s provides +a library for the requires part of the rabbitmq interface. + +However, there are two charms that provide the rabbitmq interface, being incompatible: + - https://github.com/openstack-charmers/charm-rabbitmq-ks8 (https://charmhub.io/rabbitmq-k8s) + - https://github.com/openstack/charm-rabbitmq-server/ (https://charmhub.io/rabbitmq-server) + +The main difference is that rabbitmq-server does not publish the information in the app +part in the relation bag. This python library unifies both charms, using a similar +approach to the rabbitmq-k8s library. + +For rabbitmq-k8s, the password and hostname are required in the app databag. The full +list of hostnames can be obtained from the ingress-address in each unit. + +For rabbitmq-server, the app databag is empty. The password and hostname are in the units databags, +being the password equal in all units. Each hostname may point to different addresses. One +of them will chosen as the in the rabbitmq parameters. + +rabbitmq-server support ssl client certificates, but are not implemented in this library. + +This library is very similar and uses the same events as + the library charms.rabbitmq_k8s.v0.rabbitmq. +See https://github.com/openstack-charmers/charm-rabbitmq-k8s/blob/main/lib/charms/rabbitmq_k8s/v0/rabbitmq.py # pylint: disable=line-too-long # noqa: W505 +""" + + +import logging +import urllib.parse + +from ops import CharmBase, HookEvent +from ops.framework import EventBase, EventSource, Object, ObjectEvents +from ops.model import Relation + +logger = logging.getLogger(__name__) + + +class RabbitMQConnectedEvent(EventBase): + """RabbitMQ connected Event.""" + + +class RabbitMQReadyEvent(EventBase): + """RabbitMQ ready for use Event.""" + + +class RabbitMQDepartedEvent(EventBase): + """RabbitMQ relation departed Event.""" + + +class RabbitMQServerEvents(ObjectEvents): + """Events class for `on`. + + Attributes: + connected: rabbitmq relation is connected + ready: rabbitmq relation is ready + departed: rabbitmq relation has been removed + """ + + connected = EventSource(RabbitMQConnectedEvent) + ready = EventSource(RabbitMQReadyEvent) + departed = EventSource(RabbitMQDepartedEvent) + + +class RabbitMQRequires(Object): + """RabbitMQRequires class. + + Attributes: + on: ObjectEvents for RabbitMQRequires + port: amqp port + """ + + on = RabbitMQServerEvents() + port = 5672 + + def __init__(self, charm: CharmBase, relation_name: str, username: str, vhost: str): + """Initialize the instance. + + Args: + charm: charm that uses the library + relation_name: name of the RabbitMQ relation + username: username to use for RabbitMQ + vhost: virtual host to use for RabbitMQ + """ + super().__init__(charm, relation_name) + self.charm = charm + self.relation_name = relation_name + self.username = username + self.vhost = vhost + self.framework.observe( + self.charm.on[relation_name].relation_joined, + self._on_rabbitmq_relation_joined, + ) + self.framework.observe( + self.charm.on[relation_name].relation_changed, + self._on_rabbitmq_relation_changed, + ) + self.framework.observe( + self.charm.on[relation_name].relation_departed, + self._on_rabbitmq_relation_departed, + ) + self.framework.observe( + self.charm.on[relation_name].relation_broken, + self._on_rabbitmq_relation_broken, + ) + + def _on_rabbitmq_relation_joined(self, _: HookEvent) -> None: + """Handle RabbitMQ joined.""" + self.on.connected.emit() + self.request_access(self.username, self.vhost) + + def _on_rabbitmq_relation_changed(self, _: HookEvent) -> None: + """Handle RabbitMQ changed.""" + if self.rabbitmq_uri(): + self.on.ready.emit() + + def _on_rabbitmq_relation_departed(self, _: HookEvent) -> None: + """Handle RabbitMQ departed.""" + if self.rabbitmq_uri(): + self.on.ready.emit() + + def _on_rabbitmq_relation_broken(self, _: HookEvent) -> None: + """Handle RabbitMQ broken.""" + self.on.departed.emit() + + @property + def _rabbitmq_rel(self) -> Relation | None: + """The RabbitMQ relation.""" + return self.framework.model.get_relation(self.relation_name) + + def rabbitmq_uri(self) -> str | None: + """Return RabbitMQ urs with the data in the relation. + + It will try to use the format in rabbitmq-k8s or rabbitmq-server. + If there is no relation or the data is not complete, it returns None. + + Returns: + The parameters for RabbitMQ or None. + """ + rabbitmq_k8s_params = self._rabbitmq_k8s_uri() + if rabbitmq_k8s_params: + return rabbitmq_k8s_params + + # rabbitmq-server parameters or None. + return self._rabbitmq_server_uri() + + def request_access(self, username: str, vhost: str) -> None: + """Request access to the RabbitMQ server. + + Args: + username: username requested for RabbitMQ + vhost: virtual host requested for RabbitMQ + """ + if self.model.unit.is_leader(): + if not self._rabbitmq_rel: + logger.warning("request_access but no rabbitmq relation") + return + self._rabbitmq_rel.data[self.charm.app]["username"] = username + self._rabbitmq_rel.data[self.charm.app]["vhost"] = vhost + + def _rabbitmq_server_uri(self) -> str | None: + """Return uri for rabbitmq-server. + + Returns: + Returns uri for rabbitmq-server or None if the relation data is not valid/complete. + """ + if not self._rabbitmq_rel: + return None + + password = None + hostnames = [] + for unit in self._rabbitmq_rel.units: + unit_data = self._rabbitmq_rel.data[unit] + # All of the passwords should be equal. If it is + # in the unit data, get it and override the password + password = unit_data.get("password", password) + unit_hostname = unit_data.get("hostname") + if unit_hostname: + hostnames.append(unit_hostname) + + if not password or len(hostnames) == 0: + return None + + hostname = hostnames[0] + return self._build_amqp_uri(password=password, hostname=hostname) + + def _rabbitmq_k8s_uri(self) -> str | None: + """Return URI for rabbitmq-k8s. + + Returns: + Returns uri for rabbitmq-k8s or None if the relation data is not valid/complete. + """ + if not self._rabbitmq_rel: + return None + + # A password in the _rabbitmq_rel data differentiates rabbitmq-k8s from rabbitmq-server + password = self._rabbitmq_rel.data[self._rabbitmq_rel.app].get("password") + hostname = self._rabbitmq_rel.data[self._rabbitmq_rel.app].get("hostname") + + if not password or not hostname: + return None + + return self._build_amqp_uri(password=password, hostname=hostname) + + def _build_amqp_uri(self, password: str, hostname: str) -> str: + """Return amqp URI for rabbitmq from parameters. + + Args: + password: password for amqp uri + hostname: hostname for amqp uri + + Returns: + Returns amqp uri for rabbitmq from parameters + """ + # following https://www.rabbitmq.com/docs/uri-spec#the-amqp-uri-scheme, + # vhost component of a uri should be url encoded + vhost = urllib.parse.quote(self.vhost, safe="") + return f"amqp://{self.username}:{password}@{hostname}:{self.port}/{vhost}" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0a6a41d..efb80af 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,12 +2,68 @@ # See LICENSE file for licensing details. import json +import logging import pytest import pytest_asyncio -from juju.model import Model +from juju.application import Application +from juju.client.jujudata import FileJujuData +from juju.juju import Juju +from juju.model import Controller, Model from pytest_operator.plugin import OpsTest +logger = logging.getLogger(__name__) + + +@pytest_asyncio.fixture(scope="module", name="ops_test_lxd") +async def ops_test_lxd_fixture(request, tmp_path_factory, ops_test: OpsTest): + """Return a ops_test fixture for lxd, creating the lxd controller if it does not exist.""" + if not "lxd" in Juju().get_controllers(): + logger.info("bootstrapping lxd") + _, _, _ = await ops_test.juju("bootstrap", "localhost", "lxd", check=True) + + ops_test = OpsTest(request, tmp_path_factory) + ops_test.controller_name = "lxd" + await ops_test._setup_model() + # The instance is not stored in _instance as that is done for the ops_test fixture + yield ops_test + await ops_test._cleanup_models() + + +@pytest_asyncio.fixture(scope="module", name="lxd_model") +async def lxd_model_fixture(ops_test_lxd: OpsTest) -> Model: + """Return the current lxd juju model.""" + assert ops_test_lxd.model + return ops_test_lxd.model + + +@pytest_asyncio.fixture(scope="module", name="rabbitmq_server_app") # autouse=True) +async def deploy_rabbitmq_server_fixture( + lxd_model: Model, +) -> Application: + """Deploy rabbitmq-server machine app.""" + app = await lxd_model.deploy( + "rabbitmq-server", + channel="latest/edge", + ) + await lxd_model.wait_for_idle(raise_on_blocked=True) + await lxd_model.create_offer("rabbitmq-server:amqp") + yield app + + +@pytest_asyncio.fixture(scope="module", name="rabbitmq_k8s_app") # autouse=True) +async def deploy_rabbitmq_k8s_fixture( + model: Model, +) -> Application: + """Deploy rabbitmq-k8s app.""" + app = await model.deploy( + "rabbitmq-k8s", + channel="3.12/edge", + trust=True, + ) + await model.wait_for_idle(raise_on_blocked=True) + yield app + @pytest_asyncio.fixture(scope="module", name="get_unit_ips") async def fixture_get_unit_ips(ops_test: OpsTest): diff --git a/tests/integration/flask/conftest.py b/tests/integration/flask/conftest.py index 0aa0482..d5dc9b8 100644 --- a/tests/integration/flask/conftest.py +++ b/tests/integration/flask/conftest.py @@ -252,3 +252,44 @@ def boto_s3_client_fixture(model: Model, s3_configuration: dict, s3_credentials: config=s3_client_config, ) yield s3_client + + +@pytest_asyncio.fixture(scope="function", name="rabbitmq_server_integration") +async def rabbitmq_server_integration_fixture( + ops_test_lxd: OpsTest, + flask_app: Application, + rabbitmq_server_app: Application, + model: Model, + lxd_model: Model, +): + """Integrates flask with rabbitmq-server.""" + lxd_controller = await lxd_model.get_controller() + lxd_username = lxd_controller.get_current_username() + lxd_controller_name = ops_test_lxd.controller_name + lxd_model_name = lxd_model.name + offer_name = rabbitmq_server_app.name + rabbitmq_offer_url = f"{lxd_controller_name}:{lxd_username}/{lxd_model_name}.{offer_name}" + + integration = await model.integrate(rabbitmq_offer_url, flask_app.name) + await model.wait_for_idle(apps=[flask_app.name], status="active") + + yield integration + + res = await flask_app.destroy_relation("rabbitmq", f"{rabbitmq_server_app.name}:amqp") + await model.wait_for_idle(apps=[flask_app.name], status="active") + + +@pytest_asyncio.fixture(scope="function", name="rabbitmq_k8s_integration") +async def rabbitmq_k8s_integration_fixture( + model: Model, + rabbitmq_k8s_app: Application, + flask_app: Application, +): + """Integrates flask with rabbitmq-k8s.""" + integration = await model.integrate(rabbitmq_k8s_app.name, flask_app.name) + await model.wait_for_idle(apps=[flask_app.name], status="active") + + yield integration + + await flask_app.destroy_relation("rabbitmq", f"{rabbitmq_k8s_app.name}:amqp") + await model.wait_for_idle(apps=[flask_app.name], status="active") diff --git a/tests/integration/flask/test_integrations.py b/tests/integration/flask/test_integrations.py index ad0199e..4b343fe 100644 --- a/tests/integration/flask/test_integrations.py +++ b/tests/integration/flask/test_integrations.py @@ -7,6 +7,7 @@ from secrets import token_hex import ops +import pytest import requests from juju.application import Application from juju.model import Model @@ -16,6 +17,45 @@ logger = logging.getLogger(__name__) +@pytest.mark.usefixtures("rabbitmq_server_integration") +async def test_rabbitmq_server_integration( + flask_app: Application, + get_unit_ips, +): + """ + arrange: Flask and rabbitmq-server deployed + act: Integrate flask with rabbitmq-server + assert: Assert that RabbitMQ works correctly + """ + await assert_rabbitmq_integration_correct(flask_app, get_unit_ips) + + +@pytest.mark.usefixtures("rabbitmq_k8s_integration") +async def test_rabbitmq_k8s_integration( + flask_app: Application, + get_unit_ips, +): + """ + arrange: Flask and rabbitmq-k8s deployed + act: Integrate flask with rabbitmq-k8s + assert: Assert that RabbitMQ works correctly + + """ + await assert_rabbitmq_integration_correct(flask_app, get_unit_ips) + + +async def assert_rabbitmq_integration_correct(flask_app: Application, get_unit_ips): + """Assert that rabbitmq works correctly sending and receiving a message.""" + for unit_ip in await get_unit_ips(flask_app.name): + response = requests.get(f"http://{unit_ip}:8000/rabbitmq/send", timeout=5) + assert response.status_code == 200 + assert "SUCCESS" == response.text + + response = requests.get(f"http://{unit_ip}:8000/rabbitmq/receive", timeout=5) + assert response.status_code == 200 + assert "SUCCESS" == response.text + + async def test_s3_integration( ops_test: OpsTest, flask_app: Application, diff --git a/tests/unit/flask/test_charm.py b/tests/unit/flask/test_charm.py index a1cba65..ddcad7b 100644 --- a/tests/unit/flask/test_charm.py +++ b/tests/unit/flask/test_charm.py @@ -123,7 +123,7 @@ def test_ingress(harness: Harness): def test_integrations_wiring(harness: Harness): """ - arrange: Prepare a Redis a database and a S3 integration + arrange: Prepare a Redis a database, a S3 integration and a SAML integration act: Start the flask charm and set flask-app container to be ready. assert: The flask service should have environment variables in its plan for each of the integrations. @@ -164,12 +164,120 @@ def test_integrations_wiring(harness: Harness): assert service_env["SAML_ENTITY_ID"] == SAML_APP_RELATION_DATA_EXAMPLE["entity_id"] +@pytest.mark.parametrize( + "rabbitmq_relation_data,expected_env_vars", + [ + pytest.param( + { + "app_data": { + "hostname": "rabbitmq-k8s-endpoints.testing.svc.cluster.local", + "password": "3m036hhyiDHs", + }, + "unit_data": { + "egress-subnets": "10.152.183.168/32", + "ingress-address": "10.152.183.168", + "private-address": "10.152.183.168", + }, + }, + { + "RABBITMQ_HOSTNAME": "rabbitmq-k8s-endpoints.testing.svc.cluster.local", + "RABBITMQ_USERNAME": "flask-k8s", + "RABBITMQ_PASSWORD": "3m036hhyiDHs", + "RABBITMQ_VHOST": "/", + "RABBITMQ_CONNECT_STRING": "amqp://flask-k8s:3m036hhyiDHs@rabbitmq-k8s-endpoints.testing.svc.cluster.local:5672/%2F", + }, + id="rabbitmq-k8s version", + ), + pytest.param( + { + "app_data": {}, + "unit_data": { + "hostname": "10.58.171.158", + "password": "LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8", + "private-address": "10.58.171.158", + }, + }, + { + "RABBITMQ_HOSTNAME": "10.58.171.158", + "RABBITMQ_USERNAME": "flask-k8s", + "RABBITMQ_PASSWORD": "LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8", + "RABBITMQ_VHOST": "/", + "RABBITMQ_CONNECT_STRING": "amqp://flask-k8s:LGg6HMJXPF8G3cHMcMg28ZpwSWRfS6hb8s57Jfkt5TW3rtgV5ypZkV8ZY4GcrhW8@10.58.171.158:5672/%2F", + }, + id="rabbitmq-server version", + ), + ], +) +def test_rabbitmq_integration(harness: Harness, rabbitmq_relation_data, expected_env_vars): + """ + arrange: Prepare a rabbitmq integration (RabbitMQ) + act: Start the flask charm and set flask-app container to be ready. + assert: The flask service should have environment variables in its plan + for each of the integrations. + """ + harness.add_relation("rabbitmq", "rabbitmq", **rabbitmq_relation_data) + container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) + container.add_layer("a_layer", DEFAULT_LAYER) + + harness.begin_with_initial_hooks() + + assert harness.model.unit.status == ops.ActiveStatus() + service_env = container.get_plan().services["flask"].environment + for env, env_val in expected_env_vars.items(): + assert env in service_env + assert service_env[env] == env_val + + +def test_rabbitmq_integration_with_relation_data_empty(harness: Harness): + """ + arrange: Prepare a rabbitmq integration (RabbitMQ), with missing data. + act: Start the flask charm and set flask-app container to be ready. + assert: The flask service should not have environment variables related to RabbitMQ + """ + harness.add_relation("rabbitmq", "rabbitmq") + container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) + container.add_layer("a_layer", DEFAULT_LAYER) + + harness.begin_with_initial_hooks() + + assert harness.model.unit.status == ops.ActiveStatus() + service_env = container.get_plan().services["flask"].environment + for env in service_env.keys(): + assert "RABBITMQ" not in env + + +def test_rabbitmq_remove_integration(harness: Harness): + """ + arrange: Prepare a charm with a complete rabbitmq integration (RabbitMQ). + act: Remove the relation. + assert: The relation should not have the env variables related to RabbitMQ. + """ + relation_id = harness.add_relation( + "rabbitmq", "rabbitmq", app_data={"hostname": "example.com", "password": token_hex(16)} + ) + container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) + container.add_layer("a_layer", DEFAULT_LAYER) + harness.begin_with_initial_hooks() + assert harness.model.unit.status == ops.ActiveStatus() + service_env = container.get_plan().services["flask"].environment + assert "RABBITMQ_HOSTNAME" in service_env + + harness.remove_relation(relation_id) + + service_env = container.get_plan().services["flask"].environment + assert "RABBITMQ_HOSTNAME" not in service_env + + @pytest.mark.parametrize( "integrate_to,required_integrations", [ pytest.param(["saml"], ["s3"], id="s3 fails"), pytest.param(["redis", "s3"], ["mysql", "postgresql"], id="postgresql and mysql fail"), - pytest.param([], ["mysql", "postgresql", "mongodb", "s3", "redis", "saml"], id="all fail"), + pytest.param( + [], + ["mysql", "postgresql", "mongodb", "s3", "redis", "saml", "rabbitmq"], + id="all fail", + ), ], ) def test_missing_integrations(harness: Harness, integrate_to, required_integrations): diff --git a/tests/unit/go/test_app.py b/tests/unit/go/test_app.py index 1cd9d75..4c2201b 100644 --- a/tests/unit/go/test_app.py +++ b/tests/unit/go/test_app.py @@ -32,7 +32,10 @@ {"JUJU_CHARM_HTTP_PROXY": "http://proxy.test"}, {"extra-config", "extravalue"}, {"metrics-port": "9000", "metrics-path": "/m", "app-secret-key": "notfoobar"}, - IntegrationsState(redis_uri="redis://10.1.88.132:6379"), + IntegrationsState( + redis_uri="redis://10.1.88.132:6379", + rabbitmq_uri="amqp://go-app:test-password@rabbitmq.example.com/%2f", + ), { "APP_PORT": "8080", "APP_METRICS_PATH": "/m", @@ -51,6 +54,17 @@ "APP_REDIS_DB_PORT": "6379", "APP_REDIS_DB_QUERY": "", "APP_REDIS_DB_SCHEME": "redis", + "APP_RABBITMQ_HOSTNAME": "rabbitmq.example.com", + "APP_RABBITMQ_PASSWORD": "test-password", + "APP_RABBITMQ_USERNAME": "go-app", + "APP_RABBITMQ_VHOST": "/", + "APP_RABBITMQ_CONNECT_STRING": "amqp://go-app:test-password@rabbitmq.example.com/%2f", + "APP_RABBITMQ_FRAGMENT": "", + "APP_RABBITMQ_NETLOC": "go-app:test-password@rabbitmq.example.com", + "APP_RABBITMQ_PARAMS": "", + "APP_RABBITMQ_PATH": "/%2f", + "APP_RABBITMQ_QUERY": "", + "APP_RABBITMQ_SCHEME": "amqp", }, ), ],