diff --git a/poetry.lock b/poetry.lock index d120d11d2..366f6cc9e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "allure-pytest" @@ -804,6 +804,20 @@ websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0" [package.extras] adal = ["adal (>=1.0.2)"] +[[package]] +name = "landscape-api-py3" +version = "0.9.0" +description = "Client for the Landscape API (Python 3)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "landscape_api_py3-0.9.0-py2.py3-none-any.whl", hash = "sha256:e55f7a08d3b5aca4ab0c6694ca74c0605c18fb44df8573e7a8df9e01d02d2033"}, + {file = "landscape_api_py3-0.9.0.tar.gz", hash = "sha256:83c3947ecdf4482103e37f58ea99e5e8ff3bb4370f033082bf969deaea38c762"}, +] + +[package.dependencies] +requests = "*" + [[package]] name = "macaroonbakery" version = "1.3.4" @@ -2119,4 +2133,4 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-it [metadata] lock-version = "2.0" python-versions = "^3.8.10" -content-hash = "d98fb616cf3cccb3086d37046d81600e5eff9ed7496cd57d885d5f6a1df95f97" +content-hash = "9b553dea258644c4c5d22f25283f45827b88f288040aa22eef1108a61f69352e" diff --git a/pyproject.toml b/pyproject.toml index f4d1cba3e..54997cc8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ juju = "<=3.5.0.0" tenacity = "*" mailmanclient = "^3.3.5" psycopg2-binary = "^2.9.9" +landscape-api-py3 = "^0.9.0" allure-pytest = "^2.13.5" [build-system] diff --git a/src/charm.py b/src/charm.py index 7c6f79ce9..279f346e2 100755 --- a/src/charm.py +++ b/src/charm.py @@ -104,7 +104,7 @@ def __init__(self, *args): self.legacy_db_admin_relation = DbProvides(self, admin=True) self.tls = PostgreSQLTLS(self, PEER_RELATION_NAME) - self._cores = os.cpu_count() + self._cores = max(min(os.cpu_count(), 4), 2) self.service_ids = list(range(self._cores)) self.pgb_services = [ f"{PGB}-{self.app.name}@{service_id}" for service_id in self.service_ids @@ -536,6 +536,7 @@ def generate_relation_databases(self) -> Dict[str, Dict[str, Union[str, bool]]]: return {} databases = {} + add_wildcard = False for relation in self.model.relations.get("db", []): database = self.legacy_db_relation.get_databags(relation)[0].get("database") if database: @@ -551,16 +552,22 @@ def generate_relation_databases(self) -> Dict[str, Dict[str, Union[str, bool]]]: "name": database, "legacy": True, } + add_wildcard = True for rel_id, data in self.client_relation.database_provides.fetch_relation_data( - fields=["database"] + fields=["database", "extra-user-roles"] ).items(): database = data.get("database") + roles = data.get("extra-user-roles", "").lower().split(",") if database: databases[str(rel_id)] = { "name": database, "legacy": False, } + if "admin" in roles or "superuser" in roles or "createdb" in roles: + add_wildcard = True + if add_wildcard: + databases["*"] = {"name": "*", "auth_dbname": database} self.set_relation_databases(databases) return databases @@ -586,6 +593,8 @@ def _get_relation_config(self) -> [Dict[str, Dict[str, Union[str, bool]]]]: for database in databases.values(): name = database["name"] + if name == "*": + continue pgb_dbs[name] = { "host": host, "dbname": name, @@ -599,6 +608,13 @@ def _get_relation_config(self) -> [Dict[str, Dict[str, Union[str, bool]]]]: "port": r_port, "auth_user": self.backend.auth_user, } + if "*" in databases: + pgb_dbs["*"] = { + "host": host, + "port": port, + "auth_user": self.backend.auth_user, + "auth_dbname": databases["*"]["auth_dbname"], + } return pgb_dbs def render_pgb_config(self, reload_pgbouncer=False): @@ -641,7 +657,9 @@ def render_pgb_config(self, reload_pgbouncer=False): f"{app_conf_dir}/{INSTANCE_DIR}{service_id}/pgbouncer.ini", template.render( databases=databases, - socket_dir=f"{app_temp_dir}/{INSTANCE_DIR}{service_id}", + peer_id=service_id, + base_socket_dir=f"{app_temp_dir}/{INSTANCE_DIR}", + peers=self.service_ids, log_file=f"{app_log_dir}/{INSTANCE_DIR}{service_id}/pgbouncer.log", pid_file=f"{app_temp_dir}/{INSTANCE_DIR}{service_id}/pgbouncer.pid", listen_addr=addr, diff --git a/src/relations/db.py b/src/relations/db.py index 961453046..fcd9d4333 100644 --- a/src/relations/db.py +++ b/src/relations/db.py @@ -84,6 +84,7 @@ ------------------------------------------------------------------------------------------------------------ """ # noqa: W505 +import json import logging from typing import Dict, Iterable, List @@ -107,7 +108,6 @@ MaintenanceStatus, Relation, Unit, - WaitingStatus, ) from constants import EXTENSIONS_BLOCKING_MESSAGE @@ -219,12 +219,10 @@ def _on_relation_joined(self, join_event: RelationJoinedEvent): if not (database := remote_app_databag.get("database")) and not ( database := remote_unit_databag.get("database") ): - # If there's nothing in either databag, return early. - no_db = "No database name provided in app or unit databag" - logger.warning(no_db) - self.charm.unit.status = WaitingStatus(no_db) - join_event.defer() - return + # Sometimes a relation changed event is triggered, and it doesn't have + # a database name in it (like the relation with Landscape server charm), + # so create a database with the other application name. + database = join_event.relation.app.name if self._block_on_extensions(join_event.relation, remote_app_databag): return @@ -239,16 +237,17 @@ def _on_relation_joined(self, join_event: RelationJoinedEvent): dbs = self.charm.generate_relation_databases() dbs[str(join_event.relation.id)] = {"name": database, "legacy": True} + if self.admin: + dbs["*"] = {"name": "*", "auth_dbname": database} self.charm.set_relation_databases(dbs) - self.update_databags( - join_event.relation, - { - "user": user, - "password": password, - "database": database, - }, - ) + creds = { + "user": user, + "password": password, + "database": database, + } + self.charm.peers.app_databag[user] = json.dumps(creds) + self.update_databags(join_event.relation, creds) # Create user and database in backend postgresql database try: @@ -298,7 +297,8 @@ def _on_relation_changed(self, change_event: RelationChangedEvent): # No backup values because if databag isn't populated, this relation isn't initialised. # This means that the database and user requested in this relation haven't been created, # so we defer this event until the databag is populated. - databag = self.get_databags(change_event.relation)[0] + user = self._generate_username(change_event.relation) + databag = json.loads(self.charm.peers.app_databag.get(user, "{}")) database = databag.get("database") user = databag.get("user") password = databag.get("password") @@ -311,21 +311,20 @@ def _on_relation_changed(self, change_event: RelationChangedEvent): return self.charm.render_pgb_config(reload_pgbouncer=True) - if self.charm.unit.is_leader(): - self.update_connection_info(change_event.relation, self.charm.config["listen_port"]) - self.update_databags( - change_event.relation, - { - "allowed-subnets": self.get_allowed_subnets(change_event.relation), - "allowed-units": self.get_allowed_units(change_event.relation), - "version": self.charm.backend.postgres.get_postgresql_version(), - "host": "localhost", - "user": user, - "password": password, - "database": database, - "state": self._get_state(), - }, - ) + self.update_databags( + change_event.relation, + { + "allowed-subnets": self.get_allowed_subnets(change_event.relation), + "allowed-units": self.get_allowed_units(change_event.relation), + "version": self.charm.backend.postgres.get_postgresql_version(), + "host": "localhost", + "user": user, + "password": password, + "database": database, + "state": "master", + }, + ) + self.update_connection_info(change_event.relation, self.charm.config["listen_port"]) def update_connection_info(self, relation: Relation, port: str = None): """Updates databag connection information.""" @@ -451,8 +450,7 @@ def update_databags(self, relation, updates: Dict[str, str]): for key, item in updates.items(): updates[key] = str(item) - for databag in self.get_databags(relation): - databag.update(updates) + relation.data[self.charm.unit].update(updates) def _generate_username(self, relation): """Generates a unique username for this relation.""" @@ -461,20 +459,6 @@ def _generate_username(self, relation): model_name = self.model.name return f"{app_name}_user_{relation_id}_{model_name}".replace("-", "_") - def _get_state(self) -> str: - """Gets the given state for this unit. - - Args: - standbys: the comma-separated list of postgres standbys - - Returns: - The described state of this unit. Can be 'master' or 'standby'. - """ - if self.charm.unit.is_leader(): - return "master" - else: - return "standby" - def get_allowed_subnets(self, relation: Relation) -> str: """Gets the allowed subnets from this relation.""" diff --git a/src/relations/pgbouncer_provider.py b/src/relations/pgbouncer_provider.py index 8c5f8e46e..2bbf5b24f 100644 --- a/src/relations/pgbouncer_provider.py +++ b/src/relations/pgbouncer_provider.py @@ -155,6 +155,9 @@ def _on_database_requested(self, event: DatabaseRequestedEvent) -> None: dbs = self.charm.generate_relation_databases() dbs[str(event.relation.id)] = {"name": database, "legacy": False} + roles = extra_user_roles.lower().split(",") + if "admin" in roles or "superuser" in roles: + dbs["*"] = {"name": "*", "auth_dbname": database} self.charm.set_relation_databases(dbs) # Share the credentials and updated connection info with the client application. diff --git a/templates/pgb_config.j2 b/templates/pgb_config.j2 index 4aa9d2ff1..1a7178488 100644 --- a/templates/pgb_config.j2 +++ b/templates/pgb_config.j2 @@ -1,9 +1,15 @@ [databases] -{% for name, database in databases.items() %} -{{ name }} = host={{ database.host }} dbname={{ database.dbname }} port={{ database.port }} auth_user={{ database.auth_user }} +{% for name, database in databases.items() -%} +{{ name }} = host={{ database.host }} {% if database.dbname %}dbname={{ database.dbname }}{% else %}auth_dbname={{ database.auth_dbname }}{% endif %} port={{ database.port }} auth_user={{ database.auth_user }} +{% endfor %} + +[peers] +{% for peer in peers -%} +{{ peer + 1 }} = host={{ base_socket_dir }}{{ peer }} port={{ listen_port }} {% endfor %} [pgbouncer] +peer_id = {{ peer_id + 1 }} listen_addr = {{ listen_addr }} listen_port = {{ listen_port }} logfile = {{ log_file }} @@ -15,7 +21,7 @@ max_client_conn = 10000 ignore_startup_parameters = extra_float_digits,options server_tls_sslmode = prefer so_reuseport = 1 -unix_socket_dir = {{ socket_dir }} +unix_socket_dir = {{ base_socket_dir }}{{ peer_id }} pool_mode = {{ pool_mode }} max_db_connections = {{ max_db_connections }} default_pool_size = {{ default_pool_size }} diff --git a/templates/pgbouncer.service.j2 b/templates/pgbouncer.service.j2 index cf75e9a56..a95a6a284 100644 --- a/templates/pgbouncer.service.j2 +++ b/templates/pgbouncer.service.j2 @@ -4,12 +4,10 @@ After=network.target [Service] Type=simple -# -R flag lets separate pgbouncer instances reuse sockets from previous instances, so on restart -# they'll reuse the same sockets, preserving connections. ExecStartPre=-/usr/bin/install -o snap_daemon -g snap_daemon -m 700 -d \ /var/snap/charmed-postgresql/common/var/log/pgbouncer/{{ app_name }}/instance_%i/ \ {{ snap_tmp_dir }}/{{ app_name }}/instance_%i/ -ExecStart=/snap/bin/charmed-postgresql.pgbouncer-server -R {{ conf_dir }}/{{ app_name }}/instance_%i/pgbouncer.ini +ExecStart=/snap/bin/charmed-postgresql.pgbouncer-server {{ conf_dir }}/{{ app_name }}/instance_%i/pgbouncer.ini KillSignal=SIGINT ExecReload=kill -HUP $MAINPID Restart=always diff --git a/tests/integration/helpers/postgresql_helpers.py b/tests/integration/helpers/postgresql_helpers.py index b95e8893d..cd1769ceb 100644 --- a/tests/integration/helpers/postgresql_helpers.py +++ b/tests/integration/helpers/postgresql_helpers.py @@ -1,16 +1,20 @@ #!/usr/bin/env python3 # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. +import asyncio import itertools +import os import subprocess -from typing import List, Optional +import tempfile +import zipfile +from typing import Dict, List, Optional import psycopg2 import yaml from juju.unit import Unit from pytest_operator.plugin import OpsTest -from .helpers import PG +from .helpers import PG, PGB async def build_connection_string( @@ -237,3 +241,148 @@ async def get_leader_unit(ops_test: OpsTest, app: str) -> Optional[Unit]: break return leader_unit + + +async def deploy_and_relate_bundle_with_pgbouncer( + ops_test: OpsTest, + bundle_name: str, + main_application_name: str, + main_application_num_units: int = None, + relation_name: str = "db", + status: str = "active", + status_message: str = None, + overlay: Dict = None, + timeout: int = 2000, +) -> str: + """Helper function to deploy and relate a bundle with PostgreSQL. + + Args: + ops_test: The ops test framework. + bundle_name: The name of the bundle to deploy. + main_application_name: The name of the application that should be + related to PostgreSQL. + main_application_num_units: Optional number of units for the main + application. + relation_name: The name of the relation to use in PostgreSQL + (db or db-admin). + status: Status to wait for in the application after relating + it to PostgreSQL. + status_message: Status message to wait for in the application after + relating it to PostgreSQL. + overlay: Optional overlay to be used when deploying the bundle. + timeout: Timeout to wait for the deployment to idle. + """ + # Deploy the bundle. + with tempfile.NamedTemporaryFile(dir=os.getcwd()) as original: + # Download the original bundle. + await ops_test.juju("download", bundle_name, "--filepath", original.name) + + # Open the bundle compressed file and update the contents + # of the bundle.yaml file to deploy it. + with zipfile.ZipFile(original.name, "r") as archive: + bundle_yaml = archive.read("bundle.yaml") + data = yaml.load(bundle_yaml, Loader=yaml.FullLoader) + + if main_application_num_units is not None: + data["applications"][main_application_name]["num_units"] = ( + main_application_num_units + ) + + # Save the list of relations other than `db` and `db-admin`, + # so we can add them back later. + other_relations = [ + relation for relation in data["relations"] if "postgresql" in relation + ] + + # Remove PostgreSQL and relations with it from the bundle.yaml file. + del data["applications"]["postgresql"] + data["relations"] = [ + relation + for relation in data["relations"] + if "postgresql" not in relation + and "postgresql:db" not in relation + and "postgresql:db-admin" not in relation + ] + + # Write the new bundle content to a temporary file and deploy it. + with tempfile.NamedTemporaryFile(dir=os.getcwd()) as patched: + patched.write(yaml.dump(data).encode("utf_8")) + patched.seek(0) + if overlay is not None: + with tempfile.NamedTemporaryFile() as overlay_file: + overlay_file.write(yaml.dump(overlay).encode("utf_8")) + overlay_file.seek(0) + await ops_test.juju("deploy", patched.name, "--overlay", overlay_file.name) + else: + await ops_test.juju("deploy", patched.name) + + async with ops_test.fast_forward(fast_interval="30s"): + # Relate application to PostgreSQL. + relation = await ops_test.model.relate(main_application_name, f"{PGB}:{relation_name}") + + # Restore previous existing relations. + for other_relation in other_relations: + await ops_test.model.relate(other_relation[0], other_relation[1]) + + # Wait for the deployment to complete. + unit = ops_test.model.units.get(f"{main_application_name}/0") + awaits = [ + ops_test.model.wait_for_idle( + apps=[PG, PGB], + status="active", + timeout=timeout, + ), + ops_test.model.wait_for_idle( + apps=[main_application_name], + raise_on_blocked=False, + status=status, + timeout=timeout, + ), + ] + if status_message: + awaits.append( + ops_test.model.block_until( + lambda: unit.workload_status_message == status_message, timeout=timeout + ) + ) + await asyncio.gather(*awaits) + + return relation.id + + +async def get_password(ops_test: OpsTest, unit_name: str, username: str = "operator") -> str: + """Retrieve a user password using the action. + + Args: + ops_test: ops_test instance. + unit_name: the name of the unit. + username: the user to get the password. + + Returns: + the user password. + """ + unit = ops_test.model.units.get(unit_name) + action = await unit.run_action("get-password", **{"username": username}) + result = await action.wait() + return result.results["password"] + + +async def get_landscape_api_credentials(ops_test: OpsTest) -> List[str]: + """Returns the key and secret to be used in the Landscape API. + + Args: + ops_test: The ops test framework + """ + unit = ops_test.model.applications[PG].units[0] + password = await get_password(ops_test, unit.name) + unit_address = await unit.get_public_address() + + output = await execute_query_on_unit( + unit_address, + "operator", + password, + "SELECT encode(access_key_id,'escape'), encode(access_secret_key,'escape') FROM api_credentials;", + database="landscape-standalone-main", + ) + + return output diff --git a/tests/integration/relations/test_db_admin.py b/tests/integration/relations/test_db_admin.py index c2dfd03a5..4100f9e2a 100644 --- a/tests/integration/relations/test_db_admin.py +++ b/tests/integration/relations/test_db_admin.py @@ -1,72 +1,94 @@ #!/usr/bin/env python3 # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. +import json import logging import pytest +from landscape_api.base import run_query from pytest_operator.plugin import OpsTest -from tenacity import Retrying, stop_after_attempt, wait_fixed from ..helpers.helpers import ( PG, PGB, - deploy_and_relate_application_with_pgbouncer_bundle, deploy_postgres_bundle, - get_app_relation_databag, - run_sql, + get_backend_user_pass, +) +from ..helpers.postgresql_helpers import ( + check_databases_creation, + deploy_and_relate_bundle_with_pgbouncer, + get_landscape_api_credentials, ) logger = logging.getLogger(__name__) -PSQL = "psql" -RELATION = "db-admin" +HAPROXY_APP_NAME = "haproxy" +LANDSCAPE_APP_NAME = "landscape-server" +RABBITMQ_APP_NAME = "rabbitmq-server" +DATABASE_UNITS = 2 +RELATION_NAME = "db-admin" @pytest.mark.group(1) -@pytest.mark.abort_on_fail -async def test_db_admin_with_psql(ops_test: OpsTest, pgb_charm_focal) -> None: - await deploy_postgres_bundle( +async def test_landscape_scalable_bundle_db(ops_test: OpsTest, pgb_charm_jammy: str) -> None: + """Deploy Landscape Scalable Bundle to test the 'db-admin' relation.""" + backend_relation = await deploy_postgres_bundle( ops_test, - pgb_charm_focal, - db_units=2, - pgb_series="focal", + pgb_charm_jammy, + db_units=DATABASE_UNITS, + pgb_series="jammy", + pg_config={"profile": "testing"}, + pgb_config={"max_db_connections": "20", "pool_mode": "transaction"}, ) - psql_relation = await deploy_and_relate_application_with_pgbouncer_bundle( + # Deploy and test the Landscape Scalable bundle (using this PostgreSQL charm). + await deploy_and_relate_bundle_with_pgbouncer( ops_test, - "postgresql-charmers-postgresql-client", - PSQL, - relation=RELATION, - series="focal", + "ch:landscape-scalable", + LANDSCAPE_APP_NAME, + main_application_num_units=2, + relation_name=RELATION_NAME, + timeout=3000, + ) + pgb_user, pgb_pass = await get_backend_user_pass(ops_test, backend_relation) + await check_databases_creation( + ops_test, + [ + "landscape-standalone-account-1", + "landscape-standalone-knowledge", + "landscape-standalone-main", + "landscape-standalone-package", + "landscape-standalone-resource-1", + "landscape-standalone-session", + ], + pgb_user, + pgb_pass, ) - unit_name = f"{PSQL}/0" - psql_databag = await get_app_relation_databag(ops_test, unit_name, psql_relation.id) - - pgpass = psql_databag.get("password") - user = psql_databag.get("user") - host = psql_databag.get("host") - port = psql_databag.get("port") - dbname = psql_databag.get("database") - - assert None not in [pgpass, user, host, port, dbname], "databag incorrectly populated" - - user_command = "CREATE ROLE myuser3 LOGIN PASSWORD 'mypass';" - rtn, _, err = await run_sql( - ops_test, unit_name, user_command, pgpass, user, host, port, dbname + # Create the admin user on Landscape through configs. + await ops_test.model.applications["landscape-server"].set_config({ + "admin_email": "admin@canonical.com", + "admin_name": "Admin", + "admin_password": "test1234", + }) + await ops_test.model.wait_for_idle( + apps=["landscape-server", PG, PGB], + status="active", + timeout=1200, ) - assert rtn == 0, f"failed to run admin command {user_command}, {err}" - db_command = "CREATE DATABASE test_db;" - rtn, _, err = await run_sql(ops_test, unit_name, db_command, pgpass, user, host, port, dbname) - assert rtn == 0, f"failed to run admin command {db_command}, {err}" + # Connect to the Landscape API through HAProxy and do some CRUD calls (without the update). + key, secret = await get_landscape_api_credentials(ops_test) + haproxy_unit = ops_test.model.applications[HAPROXY_APP_NAME].units[0] + api_uri = f"https://{haproxy_unit.public_address}/api/" + # Create a role and list the available roles later to check that the new one is there. + role_name = "User1" + run_query(key, secret, "CreateRole", {"name": role_name}, api_uri, False) + api_response = run_query(key, secret, "GetRoles", {}, api_uri, False) + assert role_name in [user["name"] for user in json.loads(api_response)] -@pytest.mark.group(1) -async def test_remove_relation(ops_test: OpsTest): - await ops_test.model.applications[PGB].remove_relation(f"{PGB}:db-admin", f"{PSQL}:db") - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle([PG], status="active", timeout=300) - for attempt in Retrying(stop=stop_after_attempt(60), wait=wait_fixed(1), reraise=True): - with attempt: - assert len(ops_test.model.applications[PGB].units) == 0, "pgb units were not removed" + # Remove the role and assert it isn't part of the roles list anymore. + run_query(key, secret, "RemoveRole", {"name": role_name}, api_uri, False) + api_response = run_query(key, secret, "GetRoles", {}, api_uri, False) + assert role_name not in [user["name"] for user in json.loads(api_response)] diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index f98ba9d5a..a42dc38d4 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -72,7 +72,7 @@ async def test_change_config(ops_test: OpsTest): await ops_test.model.wait_for_idle(apps=[PGB], status="active", timeout=600) # The config changes depending on the amount of cores on the unit, so get that info. - cores = await get_unit_cores(ops_test, unit) + cores = max(min(await get_unit_cores(ops_test, unit), 4), 2) primary_cfg = await get_cfg(ops_test, unit.name) @@ -90,7 +90,7 @@ async def test_change_config(ops_test: OpsTest): @pytest.mark.group(1) async def test_systemd_restarts_pgbouncer_processes(ops_test: OpsTest): unit = ops_test.model.units[f"{PGB}/0"] - expected_processes = await get_unit_cores(ops_test, unit) + expected_processes = max(min(await get_unit_cores(ops_test, unit), 4), 2) # verify the correct amount of pgbouncer processes are running assert await get_running_instances(ops_test, unit, "pgbouncer") == expected_processes diff --git a/tests/unit/relations/test_db.py b/tests/unit/relations/test_db.py index 54c231239..7e429b738 100644 --- a/tests/unit/relations/test_db.py +++ b/tests/unit/relations/test_db.py @@ -1,6 +1,7 @@ # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. +import json import unittest from unittest.mock import Mock, PropertyMock, patch, sentinel @@ -112,38 +113,44 @@ def test_on_relation_joined( _set_rel_dbs.reset_mock() self.db_admin_relation._on_relation_joined(mock_event) - _set_rel_dbs.assert_called_once_with({"1": {"name": "test_db", "legacy": True}}) + _set_rel_dbs.assert_called_once_with({ + "1": {"name": "test_db", "legacy": True}, + "*": {"name": "*", "auth_dbname": "test_db"}, + }) _create_user.assert_called_with(user, password, admin=True) _create_database.assert_called_with(database, user, client_relations=sentinel.client_rels) _init_auth.assert_called_with([database]) - for dbag in [relation_data[self.charm.unit], relation_data[self.charm.app]]: - assert dbag["database"] == database - assert dbag["user"] == user - assert dbag["password"] == password + dbag = relation_data[self.charm.unit] + assert dbag["database"] == database + assert dbag["user"] == user + assert dbag["password"] == password # Check admin permissions aren't present when we use db_relation _set_rel_dbs.reset_mock() self.db_relation._on_relation_joined(mock_event) _create_user.assert_called_with(user, password, admin=False) - _set_rel_dbs.assert_called_once_with({"1": {"name": "test_db", "legacy": True}}) + _set_rel_dbs.assert_called_once_with({ + "1": {"name": "test_db", "legacy": True}, + "*": {"name": "*", "auth_dbname": "test_db"}, + }) @patch("relations.backend_database.BackendDatabaseRequires.check_backend", return_value=True) @patch( "relations.backend_database.BackendDatabaseRequires.postgres", new_callable=PropertyMock ) - @patch("relations.db.DbProvides.get_databags", return_value=[{}]) + @patch("relations.peers.Peers.app_databag", new_callable=PropertyMock, return_value={}) @patch("relations.db.DbProvides.update_connection_info") @patch("relations.db.DbProvides.update_databags") @patch("relations.db.DbProvides.get_allowed_units") @patch("relations.db.DbProvides.get_allowed_subnets") - @patch("relations.db.DbProvides._get_state") + @patch("charm.PgBouncerCharm.get_relation_databases", return_value={"1": {"some": "data"}}) @patch("charm.PgBouncerCharm.render_pgb_config") def test_on_relation_changed( self, _render_pgb_config, - _get_state, + _, _allowed_subnets, _allowed_units, _update_databags, @@ -152,19 +159,24 @@ def test_on_relation_changed( _backend_postgres, _check_backend, ): - self.harness.set_leader(True) + with self.harness.hooks_disabled(): + self.harness.set_leader(True) + + event = Mock() + event.relation.id = 1 database = "test_db" - user = "test_user" + user = "pgbouncer_user_1_None" password = "test_pw" - _get_databags.return_value[0] = { - "database": database, - "user": user, - "password": password, + _get_databags.return_value = { + user: json.dumps({ + "database": database, + "user": user, + "password": password, + }) } # Call the function - event = Mock() self.db_relation._on_relation_changed(event) _update_connection_info.assert_called_with( @@ -180,7 +192,7 @@ def test_on_relation_changed( "user": user, "password": password, "database": database, - "state": _get_state.return_value, + "state": "master", }, ) _render_pgb_config.assert_called_once_with(reload_pgbouncer=True) @@ -236,13 +248,10 @@ def test_on_relation_departed(self, _get_units): } self.db_relation._on_relation_departed(mock_event) - app_databag = mock_event.relation.data[self.charm.app] unit_databag = mock_event.relation.data[self.charm.unit] - expected_app_databag = {"allowed-units": "test_string"} expected_unit_databag = {"allowed-units": "test_string"} - self.assertDictEqual(app_databag, expected_app_databag) self.assertDictEqual(unit_databag, expected_unit_databag) @patch("relations.backend_database.BackendDatabaseRequires.check_backend", return_value=True) diff --git a/tests/unit/relations/test_pgbouncer_provider.py b/tests/unit/relations/test_pgbouncer_provider.py index ec5a14076..5a73983db 100644 --- a/tests/unit/relations/test_pgbouncer_provider.py +++ b/tests/unit/relations/test_pgbouncer_provider.py @@ -110,7 +110,10 @@ def test_on_database_requested( _dbp_set_endpoints.assert_called_with( rel_id, f"localhost:{self.charm.config['listen_port']}" ) - _set_rel_dbs.assert_called_once_with({"1": {"name": "test-db", "legacy": False}}) + _set_rel_dbs.assert_called_once_with({ + "1": {"name": "test-db", "legacy": False}, + "*": {"name": "*", "auth_dbname": "test-db"}, + }) @patch("relations.backend_database.BackendDatabaseRequires.check_backend", return_value=True) @patch( diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index ed8fcc62e..91696cf3c 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -111,7 +111,7 @@ def test_on_install( return_value=None, ) def test_on_start(self, _has_relation, _start, _render_prom_service, _, __): - intended_instances = self._cores = os.cpu_count() + intended_instances = max(min(os.cpu_count(), 4), 2) # Testing charm blocks when systemd is in error self.charm.on.start.emit() # Charm should fail out after calling _start once @@ -140,7 +140,7 @@ def test_on_start(self, _has_relation, _start, _render_prom_service, _, __): @patch("charms.operator_libs_linux.v1.systemd.service_restart") @patch("charm.PgBouncerCharm.check_pgb_running") def test_reload_pgbouncer(self, _check_pgb_running, _restart): - intended_instances = self._cores = os.cpu_count() + intended_instances = max(min(os.cpu_count(), 4), 2) self.charm.reload_pgbouncer() calls = [call(f"pgbouncer-pgbouncer@{instance}") for instance in range(intended_instances)] _restart.assert_has_calls(calls) @@ -178,7 +178,7 @@ def test_check_pgb_running(self, _postgres_ready, _running): _running.side_effect = None # otherwise check all services and return activestatus - intended_instances = self._cores = os.cpu_count() + intended_instances = max(min(os.cpu_count(), 4), 2) assert self.charm.check_pgb_running() calls = [ call(f"pgbouncer-pgbouncer@{instance}") for instance in range(0, intended_instances) @@ -374,7 +374,9 @@ def test_render_pgb_config( for i in range(self.charm._cores): expected_content = template.render( databases=expected_databases, - socket_dir=f"/tmp/pgbouncer/instance_{i}", + peer_id=i, + peers=range(self.charm._cores), + base_socket_dir="/tmp/pgbouncer/instance_", log_file=f"{PGB_LOG_DIR}/pgbouncer/instance_{i}/pgbouncer.log", pid_file=f"/tmp/pgbouncer/instance_{i}/pgbouncer.pid", listen_addr="127.0.0.1", @@ -402,6 +404,13 @@ def test_render_pgb_config( }) del expected_databases["first_test_readonly"] del expected_databases["second_test_readonly"] + expected_databases["*"] = { + "host": "HOST", + "port": "PORT", + "auth_dbname": "first_test", + "auth_user": "pgbouncer_auth_BACKNEND_USER", + } + _get_dbs.return_value["*"] = {"name": "*", "auth_dbname": "first_test"} del _postgres_databag.return_value["read-only-endpoints"] @@ -411,7 +420,9 @@ def test_render_pgb_config( for i in range(self.charm._cores): expected_content = template.render( databases=expected_databases, - socket_dir=f"/tmp/pgbouncer/instance_{i}", + peer_id=i, + peers=range(self.charm._cores), + base_socket_dir="/tmp/pgbouncer/instance_", log_file=f"{PGB_LOG_DIR}/pgbouncer/instance_{i}/pgbouncer.log", pid_file=f"/tmp/pgbouncer/instance_{i}/pgbouncer.pid", listen_addr="127.0.0.1",