From db65025b7ed304c12bc8bd0775361bb27d0d8543 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 01:50:42 +0000 Subject: [PATCH 1/7] Update data-platform-workflows to v24 --- .github/workflows/ci.yaml | 6 +++--- .github/workflows/release.yaml | 4 ++-- .github/workflows/sync_docs.yaml | 2 +- poetry.lock | 18 +++++++++--------- pyproject.toml | 8 ++++---- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 761ee9abe..1e5c20d6e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,7 +23,7 @@ on: jobs: lint: name: Lint - uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v23.1.1 + uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.2 unit-test: name: Unit test charm @@ -45,7 +45,7 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v23.1.1 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.2 with: cache: true @@ -77,7 +77,7 @@ jobs: - lint - unit-test - build - uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v23.1.1 + uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.2 with: artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} architecture: ${{ matrix.architecture }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a21bd09b4..c83edf0d7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -25,14 +25,14 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v23.1.1 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.2 release: name: Release charm needs: - ci-tests - build - uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v23.1.1 + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.2 with: channel: 1/edge artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 03acf5ca9..ceffe91b1 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -10,7 +10,7 @@ on: jobs: sync-docs: name: Sync docs from Discourse - uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v23.1.1 + uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v24.0.2 with: reviewers: a-velasco permissions: diff --git a/poetry.lock b/poetry.lock index e2eddf81d..d43310a85 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,8 +31,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v23.1.1" -resolved_reference = "7dc172891bf274e74eef2a4d822450ca00f55188" +reference = "v24.0.2" +resolved_reference = "f92457d41a392c2549c044efe40651186db62d10" subdirectory = "python/pytest_plugins/allure_pytest_collection_report" [[package]] @@ -1873,8 +1873,8 @@ develop = false [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v23.1.1" -resolved_reference = "7dc172891bf274e74eef2a4d822450ca00f55188" +reference = "v24.0.2" +resolved_reference = "f92457d41a392c2549c044efe40651186db62d10" subdirectory = "python/pytest_plugins/github_secrets" [[package]] @@ -1911,8 +1911,8 @@ pyyaml = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v23.1.1" -resolved_reference = "7dc172891bf274e74eef2a4d822450ca00f55188" +reference = "v24.0.2" +resolved_reference = "f92457d41a392c2549c044efe40651186db62d10" subdirectory = "python/pytest_plugins/pytest_operator_cache" [[package]] @@ -1930,8 +1930,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v23.1.1" -resolved_reference = "7dc172891bf274e74eef2a4d822450ca00f55188" +reference = "v24.0.2" +resolved_reference = "f92457d41a392c2549c044efe40651186db62d10" subdirectory = "python/pytest_plugins/pytest_operator_groups" [[package]] @@ -2623,4 +2623,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.9 || >=3.10,<4" -content-hash = "697c7ce965031307b5bbbf22710120d37a4c83a9f409adfd9be922ab35063683" +content-hash = "e9b8f176f0c1ea087cd131323796ecdb7b45b87f0b3b1744340fcb1ed5b913cb" diff --git a/pyproject.toml b/pyproject.toml index 35e3b7b3c..0b4d225f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,10 +61,10 @@ optional = true [tool.poetry.group.integration.dependencies] pytest = "^8.3.4" -pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v23.1.1", subdirectory = "python/pytest_plugins/github_secrets"} +pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.2", subdirectory = "python/pytest_plugins/github_secrets"} pytest-operator = "^0.39.0" -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v23.1.1", subdirectory = "python/pytest_plugins/pytest_operator_cache"} -pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v23.1.1", subdirectory = "python/pytest_plugins/pytest_operator_groups"} +pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.2", subdirectory = "python/pytest_plugins/pytest_operator_cache"} +pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.2", subdirectory = "python/pytest_plugins/pytest_operator_groups"} # renovate caret doesn't work: https://github.com/renovatebot/renovate/issues/26940 juju = "<=3.6.1.0" tenacity = "*" @@ -72,7 +72,7 @@ mailmanclient = "^3.3.5" psycopg2-binary = "^2.9.10" landscape-api-py3 = "^0.9.0" allure-pytest = "^2.13.5" -allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v23.1.1", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} +allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.2", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} [build-system] requires = ["poetry-core>=1.0.0"] From 1625c6b8a7814f4137f778666b41f78a392bdf44 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Mon, 6 Jan 2025 17:46:51 +0200 Subject: [PATCH 2/7] Bump libs --- lib/charms/grafana_agent/v0/cos_agent.py | 3 +- lib/charms/operator_libs_linux/v2/snap.py | 45 +++- lib/charms/postgresql_k8s/v0/postgresql.py | 195 +++++++++--------- .../postgresql_k8s/v0/postgresql_tls.py | 7 +- .../v2/tls_certificates.py | 4 +- 5 files changed, 141 insertions(+), 113 deletions(-) diff --git a/lib/charms/grafana_agent/v0/cos_agent.py b/lib/charms/grafana_agent/v0/cos_agent.py index cc4da25a8..1ea79a625 100644 --- a/lib/charms/grafana_agent/v0/cos_agent.py +++ b/lib/charms/grafana_agent/v0/cos_agent.py @@ -22,7 +22,6 @@ Using the `COSAgentProvider` object only requires instantiating it, typically in the `__init__` method of your charm (the one which sends telemetry). -The constructor of `COSAgentProvider` has only one required and ten optional parameters: ```python def __init__( @@ -253,7 +252,7 @@ class _MetricsEndpointDict(TypedDict): LIBID = "dc15fa84cef84ce58155fb84f6c6213a" LIBAPI = 0 -LIBPATCH = 11 +LIBPATCH = 12 PYDEPS = ["cosl", "pydantic"] diff --git a/lib/charms/operator_libs_linux/v2/snap.py b/lib/charms/operator_libs_linux/v2/snap.py index 9d09a78d3..d14f864fd 100644 --- a/lib/charms/operator_libs_linux/v2/snap.py +++ b/lib/charms/operator_libs_linux/v2/snap.py @@ -64,6 +64,7 @@ import socket import subprocess import sys +import time import urllib.error import urllib.parse import urllib.request @@ -83,7 +84,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 7 +LIBPATCH = 9 # Regex to locate 7-bit C1 ANSI sequences @@ -332,7 +333,7 @@ def get(self, key: Optional[str], *, typed: bool = False) -> Any: return self._snap("get", [key]).strip() - def set(self, config: Dict[str, Any], *, typed: bool = False) -> str: + def set(self, config: Dict[str, Any], *, typed: bool = False) -> None: """Set a snap configuration value. Args: @@ -340,11 +341,9 @@ def set(self, config: Dict[str, Any], *, typed: bool = False) -> str: typed: set to True to convert all values in the config into typed values while configuring the snap (set with typed=True). Default is not to convert. """ - if typed: - kv = [f"{key}={json.dumps(val)}" for key, val in config.items()] - return self._snap("set", ["-t"] + kv) - - return self._snap("set", [f"{key}={val}" for key, val in config.items()]) + if not typed: + config = {k: str(v) for k, v in config.items()} + self._snap_client._put_snap_conf(self._name, config) def unset(self, key) -> str: """Unset a snap configuration value. @@ -770,7 +769,33 @@ def _request( headers["Content-Type"] = "application/json" response = self._request_raw(method, path, query, headers, data) - return json.loads(response.read().decode())["result"] + response = json.loads(response.read().decode()) + if response["type"] == "async": + return self._wait(response["change"]) + return response["result"] + + def _wait(self, change_id: str, timeout=300) -> JSONType: + """Wait for an async change to complete. + + The poll time is 100 milliseconds, the same as in snap clients. + """ + deadline = time.time() + timeout + while True: + if time.time() > deadline: + raise TimeoutError(f"timeout waiting for snap change {change_id}") + response = self._request("GET", f"changes/{change_id}") + status = response["status"] + if status == "Done": + return response.get("data") + if status == "Doing" or status == "Do": + time.sleep(0.1) + continue + if status == "Wait": + logger.warning("snap change %s succeeded with status 'Wait'", change_id) + return response.get("data") + raise SnapError( + f"snap change {response.get('kind')!r} id {change_id} failed with status {status}" + ) def _request_raw( self, @@ -818,6 +843,10 @@ def get_installed_snap_apps(self, name: str) -> List: """Query the snap server for apps belonging to a named, currently installed snap.""" return self._request("GET", "apps", {"names": name, "select": "service"}) + def _put_snap_conf(self, name: str, conf: Dict[str, Any]): + """Set the configuration details for an installed snap.""" + return self._request("PUT", f"snaps/{name}/conf", body=conf) + class SnapCache(Mapping): """An abstraction to represent installed/available packages. diff --git a/lib/charms/postgresql_k8s/v0/postgresql.py b/lib/charms/postgresql_k8s/v0/postgresql.py index 4d8d6dc30..17adae9e6 100644 --- a/lib/charms/postgresql_k8s/v0/postgresql.py +++ b/lib/charms/postgresql_k8s/v0/postgresql.py @@ -21,12 +21,11 @@ import logging from collections import OrderedDict -from typing import Dict, List, Optional, Set, Tuple +from typing import Optional, Set, Tuple import psycopg2 from ops.model import Relation -from psycopg2 import sql -from psycopg2.sql import Composed +from psycopg2.sql import SQL, Composed, Identifier, Literal # The unique Charmhub library identifier, never change it LIBID = "24ee217a54e840a598ff21a079c3e678" @@ -36,7 +35,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 39 +LIBPATCH = 40 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles" @@ -62,7 +61,7 @@ class PostgreSQLCreateDatabaseError(Exception): class PostgreSQLCreateUserError(Exception): """Exception raised when creating a user fails.""" - def __init__(self, message: str = None): + def __init__(self, message: Optional[str] = None): super().__init__(message) self.message = message @@ -109,14 +108,14 @@ def __init__( user: str, password: str, database: str, - system_users: List[str] = [], + system_users: Optional[list[str]] = None, ): self.primary_host = primary_host self.current_host = current_host self.user = user self.password = password self.database = database - self.system_users = system_users + self.system_users = system_users if system_users else [] def _configure_pgaudit(self, enable: bool) -> None: connection = None @@ -138,7 +137,7 @@ def _configure_pgaudit(self, enable: bool) -> None: connection.close() def _connect_to_database( - self, database: str = None, database_host: str = None + self, database: Optional[str] = None, database_host: Optional[str] = None ) -> psycopg2.extensions.connection: """Creates a connection to the database. @@ -162,8 +161,8 @@ def create_database( self, database: str, user: str, - plugins: List[str] = [], - client_relations: List[Relation] = [], + plugins: Optional[list[str]] = None, + client_relations: Optional[list[Relation]] = None, ) -> None: """Creates a new database and grant privileges to a user on it. @@ -173,21 +172,25 @@ def create_database( plugins: extensions to enable in the new database. client_relations: current established client relations. """ + plugins = plugins if plugins else [] + client_relations = client_relations if client_relations else [] try: connection = self._connect_to_database() cursor = connection.cursor() - cursor.execute(f"SELECT datname FROM pg_database WHERE datname='{database}';") + cursor.execute( + SQL("SELECT datname FROM pg_database WHERE datname={};").format(Literal(database)) + ) if cursor.fetchone() is None: - cursor.execute(sql.SQL("CREATE DATABASE {};").format(sql.Identifier(database))) + cursor.execute(SQL("CREATE DATABASE {};").format(Identifier(database))) cursor.execute( - sql.SQL("REVOKE ALL PRIVILEGES ON DATABASE {} FROM PUBLIC;").format( - sql.Identifier(database) + SQL("REVOKE ALL PRIVILEGES ON DATABASE {} FROM PUBLIC;").format( + Identifier(database) ) ) - for user_to_grant_access in [user, "admin"] + self.system_users: + for user_to_grant_access in [user, "admin", *self.system_users]: cursor.execute( - sql.SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {};").format( - sql.Identifier(database), sql.Identifier(user_to_grant_access) + SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {};").format( + Identifier(database), Identifier(user_to_grant_access) ) ) relations_accessing_this_database = 0 @@ -195,26 +198,29 @@ def create_database( for data in relation.data.values(): if data.get("database") == database: relations_accessing_this_database += 1 - with self._connect_to_database(database=database) as conn: - with conn.cursor() as curs: - curs.execute( - "SELECT schema_name FROM information_schema.schemata WHERE schema_name NOT LIKE 'pg_%' and schema_name <> 'information_schema';" - ) - schemas = [row[0] for row in curs.fetchall()] - statements = self._generate_database_privileges_statements( - relations_accessing_this_database, schemas, user - ) - for statement in statements: - curs.execute(statement) + with self._connect_to_database(database=database) as conn, conn.cursor() as curs: + curs.execute( + "SELECT schema_name FROM information_schema.schemata WHERE schema_name NOT LIKE 'pg_%' and schema_name <> 'information_schema';" + ) + schemas = [row[0] for row in curs.fetchall()] + statements = self._generate_database_privileges_statements( + relations_accessing_this_database, schemas, user + ) + for statement in statements: + curs.execute(statement) except psycopg2.Error as e: logger.error(f"Failed to create database: {e}") - raise PostgreSQLCreateDatabaseError() + raise PostgreSQLCreateDatabaseError() from e # Enable preset extensions self.enable_disable_extensions({plugin: True for plugin in plugins}, database) def create_user( - self, user: str, password: str = None, admin: bool = False, extra_user_roles: str = None + self, + user: str, + password: Optional[str] = None, + admin: bool = False, + extra_user_roles: Optional[str] = None, ) -> None: """Creates a database user. @@ -249,7 +255,9 @@ def create_user( with self._connect_to_database() as connection, connection.cursor() as cursor: # Create or update the user. - cursor.execute(f"SELECT TRUE FROM pg_roles WHERE rolname='{user}';") + cursor.execute( + SQL("SELECT TRUE FROM pg_roles WHERE rolname={};").format(Literal(user)) + ) if cursor.fetchone() is not None: user_definition = "ALTER ROLE {}" else: @@ -257,22 +265,20 @@ def create_user( user_definition += f"WITH {'NOLOGIN' if user == 'admin' else 'LOGIN'}{' SUPERUSER' if admin else ''} ENCRYPTED PASSWORD '{password}'{'IN ROLE admin CREATEDB' if admin_role else ''}" if privileges: user_definition += f" {' '.join(privileges)}" - cursor.execute(sql.SQL("BEGIN;")) - cursor.execute(sql.SQL("SET LOCAL log_statement = 'none';")) - cursor.execute(sql.SQL(f"{user_definition};").format(sql.Identifier(user))) - cursor.execute(sql.SQL("COMMIT;")) + cursor.execute(SQL("BEGIN;")) + cursor.execute(SQL("SET LOCAL log_statement = 'none';")) + cursor.execute(SQL(f"{user_definition};").format(Identifier(user))) + cursor.execute(SQL("COMMIT;")) # Add extra user roles to the new user. if roles: for role in roles: cursor.execute( - sql.SQL("GRANT {} TO {};").format( - sql.Identifier(role), sql.Identifier(user) - ) + SQL("GRANT {} TO {};").format(Identifier(role), Identifier(user)) ) except psycopg2.Error as e: logger.error(f"Failed to create user: {e}") - raise PostgreSQLCreateUserError() + raise PostgreSQLCreateUserError() from e def delete_user(self, user: str) -> None: """Deletes a database user. @@ -298,20 +304,22 @@ def delete_user(self, user: str) -> None: database ) as connection, connection.cursor() as cursor: cursor.execute( - sql.SQL("REASSIGN OWNED BY {} TO {};").format( - sql.Identifier(user), sql.Identifier(self.user) + SQL("REASSIGN OWNED BY {} TO {};").format( + Identifier(user), Identifier(self.user) ) ) - cursor.execute(sql.SQL("DROP OWNED BY {};").format(sql.Identifier(user))) + cursor.execute(SQL("DROP OWNED BY {};").format(Identifier(user))) # Delete the user. with self._connect_to_database() as connection, connection.cursor() as cursor: - cursor.execute(sql.SQL("DROP ROLE {};").format(sql.Identifier(user))) + cursor.execute(SQL("DROP ROLE {};").format(Identifier(user))) except psycopg2.Error as e: logger.error(f"Failed to delete user: {e}") - raise PostgreSQLDeleteUserError() + raise PostgreSQLDeleteUserError() from e - def enable_disable_extensions(self, extensions: Dict[str, bool], database: str = None) -> None: + def enable_disable_extensions( + self, extensions: dict[str, bool], database: Optional[str] = None + ) -> None: """Enables or disables a PostgreSQL extension. Args: @@ -353,20 +361,20 @@ def enable_disable_extensions(self, extensions: Dict[str, bool], database: str = pass except psycopg2.errors.DependentObjectsStillExist: raise - except psycopg2.Error: - raise PostgreSQLEnableDisableExtensionError() + except psycopg2.Error as e: + raise PostgreSQLEnableDisableExtensionError() from e finally: if connection is not None: connection.close() def _generate_database_privileges_statements( - self, relations_accessing_this_database: int, schemas: List[str], user: str - ) -> List[Composed]: + self, relations_accessing_this_database: int, schemas: list[str], user: str + ) -> list[Composed]: """Generates a list of databases privileges statements.""" statements = [] if relations_accessing_this_database == 1: statements.append( - sql.SQL( + SQL( """DO $$ DECLARE r RECORD; BEGIN @@ -386,44 +394,42 @@ def _generate_database_privileges_statements( END LOOP; END; $$;""" ).format( - sql.Identifier(user), - sql.Identifier(user), - sql.Identifier(user), - sql.Identifier(user), - sql.Identifier(user), - sql.Identifier(user), + Identifier(user), + Identifier(user), + Identifier(user), + Identifier(user), + Identifier(user), + Identifier(user), ) ) statements.append( - """UPDATE pg_catalog.pg_largeobject_metadata -SET lomowner = (SELECT oid FROM pg_roles WHERE rolname = '{}') -WHERE lomowner = (SELECT oid FROM pg_roles WHERE rolname = '{}');""".format(user, self.user) + SQL( + "UPDATE pg_catalog.pg_largeobject_metadata\n" + "SET lomowner = (SELECT oid FROM pg_roles WHERE rolname = {})\n" + "WHERE lomowner = (SELECT oid FROM pg_roles WHERE rolname = {});" + ).format(Literal(user), Literal(self.user)) ) for schema in schemas: statements.append( - sql.SQL("ALTER SCHEMA {} OWNER TO {};").format( - sql.Identifier(schema), sql.Identifier(user) + SQL("ALTER SCHEMA {} OWNER TO {};").format( + Identifier(schema), Identifier(user) ) ) else: for schema in schemas: - schema = sql.Identifier(schema) + schema = Identifier(schema) statements.extend([ - sql.SQL("GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA {} TO {};").format( - schema, sql.Identifier(user) - ), - sql.SQL("GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA {} TO {};").format( - schema, sql.Identifier(user) + SQL("GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA {} TO {};").format( + schema, Identifier(user) ), - sql.SQL("GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA {} TO {};").format( - schema, sql.Identifier(user) + SQL("GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA {} TO {};").format( + schema, Identifier(user) ), - sql.SQL("GRANT USAGE ON SCHEMA {} TO {};").format( - schema, sql.Identifier(user) - ), - sql.SQL("GRANT CREATE ON SCHEMA {} TO {};").format( - schema, sql.Identifier(user) + SQL("GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA {} TO {};").format( + schema, Identifier(user) ), + SQL("GRANT USAGE ON SCHEMA {} TO {};").format(schema, Identifier(user)), + SQL("GRANT CREATE ON SCHEMA {} TO {};").format(schema, Identifier(user)), ]) return statements @@ -435,7 +441,7 @@ def get_last_archived_wal(self) -> str: return cursor.fetchone()[0] except psycopg2.Error as e: logger.error(f"Failed to get PostgreSQL last archived WAL: {e}") - raise PostgreSQLGetLastArchivedWALError() + raise PostgreSQLGetLastArchivedWALError() from e def get_current_timeline(self) -> str: """Get the timeline id for the current PostgreSQL unit.""" @@ -445,7 +451,7 @@ def get_current_timeline(self) -> str: return cursor.fetchone()[0] except psycopg2.Error as e: logger.error(f"Failed to get PostgreSQL current timeline id: {e}") - raise PostgreSQLGetCurrentTimelineError() + raise PostgreSQLGetCurrentTimelineError() from e def get_postgresql_text_search_configs(self) -> Set[str]: """Returns the PostgreSQL available text search configs. @@ -479,10 +485,7 @@ def get_postgresql_version(self, current_host=True) -> str: Returns: PostgreSQL version number. """ - if current_host: - host = self.current_host - else: - host = None + host = self.current_host if current_host else None try: with self._connect_to_database( database_host=host @@ -492,7 +495,7 @@ def get_postgresql_version(self, current_host=True) -> str: return cursor.fetchone()[0].split(" ")[1] except psycopg2.Error as e: logger.error(f"Failed to get PostgreSQL version: {e}") - raise PostgreSQLGetPostgreSQLVersionError() + raise PostgreSQLGetPostgreSQLVersionError() from e def is_tls_enabled(self, check_current_host: bool = False) -> bool: """Returns whether TLS is enabled. @@ -527,7 +530,7 @@ def list_users(self) -> Set[str]: return {username[0] for username in usernames} except psycopg2.Error as e: logger.error(f"Failed to list PostgreSQL database users: {e}") - raise PostgreSQLListUsersError() + raise PostgreSQLListUsersError() from e def list_valid_privileges_and_roles(self) -> Tuple[Set[str], Set[str]]: """Returns two sets with valid privileges and roles. @@ -558,8 +561,8 @@ def set_up_database(self) -> None: cursor.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC;") for user in self.system_users: cursor.execute( - sql.SQL("GRANT ALL PRIVILEGES ON DATABASE postgres TO {};").format( - sql.Identifier(user) + SQL("GRANT ALL PRIVILEGES ON DATABASE postgres TO {};").format( + Identifier(user) ) ) self.create_user( @@ -569,13 +572,13 @@ def set_up_database(self) -> None: cursor.execute("GRANT CONNECT ON DATABASE postgres TO admin;") except psycopg2.Error as e: logger.error(f"Failed to set up databases: {e}") - raise PostgreSQLDatabasesSetupError() + raise PostgreSQLDatabasesSetupError() from e finally: if connection is not None: connection.close() def update_user_password( - self, username: str, password: str, database_host: str = None + self, username: str, password: str, database_host: Optional[str] = None ) -> None: """Update a user password. @@ -592,17 +595,17 @@ def update_user_password( with self._connect_to_database( database_host=database_host ) as connection, connection.cursor() as cursor: - cursor.execute(sql.SQL("BEGIN;")) - cursor.execute(sql.SQL("SET LOCAL log_statement = 'none';")) + cursor.execute(SQL("BEGIN;")) + cursor.execute(SQL("SET LOCAL log_statement = 'none';")) cursor.execute( - sql.SQL("ALTER USER {} WITH ENCRYPTED PASSWORD '" + password + "';").format( - sql.Identifier(username) + SQL("ALTER USER {} WITH ENCRYPTED PASSWORD '" + password + "';").format( + Identifier(username) ) ) - cursor.execute(sql.SQL("COMMIT;")) + cursor.execute(SQL("COMMIT;")) except psycopg2.Error as e: logger.error(f"Failed to update user password: {e}") - raise PostgreSQLUpdateUserPasswordError() + raise PostgreSQLUpdateUserPasswordError() from e finally: if connection is not None: connection.close() @@ -626,8 +629,8 @@ def is_restart_pending(self) -> bool: @staticmethod def build_postgresql_parameters( - config_options: Dict, available_memory: int, limit_memory: Optional[int] = None - ) -> Optional[Dict]: + config_options: dict, available_memory: int, limit_memory: Optional[int] = None + ) -> Optional[dict]: """Builds the PostgreSQL parameters. Args: @@ -692,9 +695,9 @@ def validate_date_style(self, date_style: str) -> bool: database_host=self.current_host ) as connection, connection.cursor() as cursor: cursor.execute( - sql.SQL( + SQL( "SET DateStyle to {};", - ).format(sql.Identifier(date_style)) + ).format(Identifier(date_style)) ) return True except psycopg2.Error: diff --git a/lib/charms/postgresql_k8s/v0/postgresql_tls.py b/lib/charms/postgresql_k8s/v0/postgresql_tls.py index 9c5184ef1..bdc7159a9 100644 --- a/lib/charms/postgresql_k8s/v0/postgresql_tls.py +++ b/lib/charms/postgresql_k8s/v0/postgresql_tls.py @@ -45,7 +45,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version. -LIBPATCH = 10 +LIBPATCH = 11 logger = logging.getLogger(__name__) SCOPE = "unit" @@ -82,10 +82,7 @@ def _on_set_tls_private_key(self, event: ActionEvent) -> None: def _request_certificate(self, param: Optional[str]): """Request a certificate to TLS Certificates Operator.""" - if param is None: - key = generate_private_key() - else: - key = self._parse_tls_file(param) + key = generate_private_key() if param is None else self._parse_tls_file(param) csr = generate_csr( private_key=key, diff --git a/lib/charms/tls_certificates_interface/v2/tls_certificates.py b/lib/charms/tls_certificates_interface/v2/tls_certificates.py index 9f67833ba..c232362fe 100644 --- a/lib/charms/tls_certificates_interface/v2/tls_certificates.py +++ b/lib/charms/tls_certificates_interface/v2/tls_certificates.py @@ -307,7 +307,7 @@ def _on_all_certificates_invalidated(self, event: AllCertificatesInvalidatedEven # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 28 +LIBPATCH = 29 PYDEPS = ["cryptography", "jsonschema"] @@ -459,7 +459,7 @@ def restore(self, snapshot: dict): class CertificateExpiringEvent(EventBase): """Charm Event triggered when a TLS certificate is almost expired.""" - def __init__(self, handle, certificate: str, expiry: str): + def __init__(self, handle: Handle, certificate: str, expiry: str): """CertificateExpiringEvent. Args: From db647e8d853627ca3ec263fd32d064247dde97b3 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Mon, 6 Jan 2025 17:50:37 +0200 Subject: [PATCH 3/7] Disable juju 3.6 --- .github/workflows/ci.yaml | 15 ++++----------- .github/workflows/release.yaml | 4 ++-- .github/workflows/sync_docs.yaml | 2 +- charmcraft.yaml | 1 + poetry.lock | 22 ++++++++++++---------- pyproject.toml | 8 ++++---- 6 files changed, 24 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1e5c20d6e..87bfb52bf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,7 +23,7 @@ on: jobs: lint: name: Lint - uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.2 + uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.5 unit-test: name: Unit test charm @@ -45,7 +45,7 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.2 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.5 with: cache: true @@ -59,8 +59,6 @@ jobs: allure_on_amd64: false - agent: 3.4.6 # renovate: juju-agent-pin-minor allure_on_amd64: true - - snap_channel: 3.6/stable - allure_on_amd64: false architecture: - amd64 include: @@ -68,22 +66,17 @@ jobs: agent: 3.4.6 # renovate: juju-agent-pin-minor allure_on_amd64: true architecture: arm64 - - juju: - snap_channel: 3.6/stable - allure_on_amd64: false - architecture: arm64 - name: Integration | ${{ matrix.juju.agent || matrix.juju.snap_channel }} | ${{ matrix.architecture }} + name: Integration | ${{ matrix.juju.agent }} | ${{ matrix.architecture }} needs: - lint - unit-test - build - uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.2 + uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.5 with: artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} architecture: ${{ matrix.architecture }} cloud: lxd juju-agent-version: ${{ matrix.juju.agent }} - juju-snap-channel: ${{ matrix.juju.snap_channel }} libjuju-version-constraint: ${{ matrix.juju.libjuju }} _beta_allure_report: ${{ matrix.juju.allure_on_amd64 && matrix.architecture == 'amd64' }} permissions: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c83edf0d7..8975e6fb3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -25,14 +25,14 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.2 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.5 release: name: Release charm needs: - ci-tests - build - uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.2 + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.5 with: channel: 1/edge artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index ceffe91b1..04ce0b134 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -10,7 +10,7 @@ on: jobs: sync-docs: name: Sync docs from Discourse - uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v24.0.2 + uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v24.0.5 with: reviewers: a-velasco permissions: diff --git a/charmcraft.yaml b/charmcraft.yaml index c438f0100..cc89f571d 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -30,6 +30,7 @@ parts: # Convert subset of poetry.lock to requirements.txt curl -sSL https://install.python-poetry.org | python3 - + /root/.local/bin/poetry self add poetry-plugin-export /root/.local/bin/poetry export --only main,charm-libs --output requirements.txt craftctl default diff --git a/poetry.lock b/poetry.lock index d43310a85..d57f18ac5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "allure-pytest" @@ -31,8 +31,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.2" -resolved_reference = "f92457d41a392c2549c044efe40651186db62d10" +reference = "v24.0.5" +resolved_reference = "078fb649213bbffe223e826f04560c2e9c9c3396" subdirectory = "python/pytest_plugins/allure_pytest_collection_report" [[package]] @@ -1551,6 +1551,7 @@ files = [ {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, + {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, @@ -1610,6 +1611,7 @@ files = [ {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909"}, {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1"}, {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142"}, {file = "psycopg2_binary-2.9.10-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4"}, {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8"}, {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864"}, @@ -1873,8 +1875,8 @@ develop = false [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.2" -resolved_reference = "f92457d41a392c2549c044efe40651186db62d10" +reference = "v24.0.5" +resolved_reference = "078fb649213bbffe223e826f04560c2e9c9c3396" subdirectory = "python/pytest_plugins/github_secrets" [[package]] @@ -1911,8 +1913,8 @@ pyyaml = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.2" -resolved_reference = "f92457d41a392c2549c044efe40651186db62d10" +reference = "v24.0.5" +resolved_reference = "078fb649213bbffe223e826f04560c2e9c9c3396" subdirectory = "python/pytest_plugins/pytest_operator_cache" [[package]] @@ -1930,8 +1932,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.2" -resolved_reference = "f92457d41a392c2549c044efe40651186db62d10" +reference = "v24.0.5" +resolved_reference = "078fb649213bbffe223e826f04560c2e9c9c3396" subdirectory = "python/pytest_plugins/pytest_operator_groups" [[package]] @@ -2623,4 +2625,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.9 || >=3.10,<4" -content-hash = "e9b8f176f0c1ea087cd131323796ecdb7b45b87f0b3b1744340fcb1ed5b913cb" +content-hash = "fb4f6ac599bbc8ff8cae96e65f6a99f30bb2e526ef5b17b6abeb62a0771f901a" diff --git a/pyproject.toml b/pyproject.toml index 0b4d225f7..4a88f279b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,10 +61,10 @@ optional = true [tool.poetry.group.integration.dependencies] pytest = "^8.3.4" -pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.2", subdirectory = "python/pytest_plugins/github_secrets"} +pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.5", subdirectory = "python/pytest_plugins/github_secrets"} pytest-operator = "^0.39.0" -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.2", subdirectory = "python/pytest_plugins/pytest_operator_cache"} -pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.2", subdirectory = "python/pytest_plugins/pytest_operator_groups"} +pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.5", subdirectory = "python/pytest_plugins/pytest_operator_cache"} +pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.5", subdirectory = "python/pytest_plugins/pytest_operator_groups"} # renovate caret doesn't work: https://github.com/renovatebot/renovate/issues/26940 juju = "<=3.6.1.0" tenacity = "*" @@ -72,7 +72,7 @@ mailmanclient = "^3.3.5" psycopg2-binary = "^2.9.10" landscape-api-py3 = "^0.9.0" allure-pytest = "^2.13.5" -allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.2", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} +allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.5", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} [build-system] requires = ["poetry-core>=1.0.0"] From fd26cdeeefe1b6c845e5e8b03962907d8f0d28d4 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Tue, 7 Jan 2025 13:13:35 +0200 Subject: [PATCH 4/7] Focal poetry --- charmcraft.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charmcraft.yaml b/charmcraft.yaml index cc89f571d..d58b3d453 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -29,8 +29,7 @@ parts: rustup default stable # Convert subset of poetry.lock to requirements.txt - curl -sSL https://install.python-poetry.org | python3 - - /root/.local/bin/poetry self add poetry-plugin-export + curl -sSL https://install.python-poetry.org | python3 - --version 1.8.5 /root/.local/bin/poetry self add poetry-plugin-export /root/.local/bin/poetry export --only main,charm-libs --output requirements.txt craftctl default From 15fd3d9f8015d8837811a65d56553d345ccc9a05 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Tue, 7 Jan 2025 14:09:26 +0200 Subject: [PATCH 5/7] Bump dpw --- .github/workflows/ci.yaml | 6 +++--- .github/workflows/release.yaml | 4 ++-- .github/workflows/sync_docs.yaml | 2 +- poetry.lock | 18 +++++++++--------- pyproject.toml | 8 ++++---- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 87bfb52bf..124f5e1ad 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,7 +23,7 @@ on: jobs: lint: name: Lint - uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.5 + uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.6 unit-test: name: Unit test charm @@ -45,7 +45,7 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.5 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6 with: cache: true @@ -71,7 +71,7 @@ jobs: - lint - unit-test - build - uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.5 + uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.6 with: artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} architecture: ${{ matrix.architecture }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8975e6fb3..32265636d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -25,14 +25,14 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.5 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6 release: name: Release charm needs: - ci-tests - build - uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.5 + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.6 with: channel: 1/edge artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 04ce0b134..b31f67bcd 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -10,7 +10,7 @@ on: jobs: sync-docs: name: Sync docs from Discourse - uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v24.0.5 + uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v24.0.6 with: reviewers: a-velasco permissions: diff --git a/poetry.lock b/poetry.lock index d57f18ac5..034cd99a9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,8 +31,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.5" -resolved_reference = "078fb649213bbffe223e826f04560c2e9c9c3396" +reference = "v24.0.6" +resolved_reference = "11c673f692893a15d15ee63469420e91f91f8a95" subdirectory = "python/pytest_plugins/allure_pytest_collection_report" [[package]] @@ -1875,8 +1875,8 @@ develop = false [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.5" -resolved_reference = "078fb649213bbffe223e826f04560c2e9c9c3396" +reference = "v24.0.6" +resolved_reference = "11c673f692893a15d15ee63469420e91f91f8a95" subdirectory = "python/pytest_plugins/github_secrets" [[package]] @@ -1913,8 +1913,8 @@ pyyaml = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.5" -resolved_reference = "078fb649213bbffe223e826f04560c2e9c9c3396" +reference = "v24.0.6" +resolved_reference = "11c673f692893a15d15ee63469420e91f91f8a95" subdirectory = "python/pytest_plugins/pytest_operator_cache" [[package]] @@ -1932,8 +1932,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.5" -resolved_reference = "078fb649213bbffe223e826f04560c2e9c9c3396" +reference = "v24.0.6" +resolved_reference = "11c673f692893a15d15ee63469420e91f91f8a95" subdirectory = "python/pytest_plugins/pytest_operator_groups" [[package]] @@ -2625,4 +2625,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.9 || >=3.10,<4" -content-hash = "fb4f6ac599bbc8ff8cae96e65f6a99f30bb2e526ef5b17b6abeb62a0771f901a" +content-hash = "e5f6f3f91c594e10f087c41ae4e62c190403da3772e1c9d56ed46cd892584c19" diff --git a/pyproject.toml b/pyproject.toml index 4a88f279b..0ea81be97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,10 +61,10 @@ optional = true [tool.poetry.group.integration.dependencies] pytest = "^8.3.4" -pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.5", subdirectory = "python/pytest_plugins/github_secrets"} +pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/github_secrets"} pytest-operator = "^0.39.0" -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.5", subdirectory = "python/pytest_plugins/pytest_operator_cache"} -pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.5", subdirectory = "python/pytest_plugins/pytest_operator_groups"} +pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_cache"} +pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_groups"} # renovate caret doesn't work: https://github.com/renovatebot/renovate/issues/26940 juju = "<=3.6.1.0" tenacity = "*" @@ -72,7 +72,7 @@ mailmanclient = "^3.3.5" psycopg2-binary = "^2.9.10" landscape-api-py3 = "^0.9.0" allure-pytest = "^2.13.5" -allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.5", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} +allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} [build-system] requires = ["poetry-core>=1.0.0"] From 597040b99124af68b65302795a6194b7c2a1237b Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Tue, 7 Jan 2025 14:17:18 +0200 Subject: [PATCH 6/7] Typo --- charmcraft.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charmcraft.yaml b/charmcraft.yaml index d58b3d453..dfad2e9f0 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -29,7 +29,8 @@ parts: rustup default stable # Convert subset of poetry.lock to requirements.txt - curl -sSL https://install.python-poetry.org | python3 - --version 1.8.5 /root/.local/bin/poetry self add poetry-plugin-export + curl -sSL https://install.python-poetry.org | python3 - --version 1.8.5 + /root/.local/bin/poetry self add poetry-plugin-export /root/.local/bin/poetry export --only main,charm-libs --output requirements.txt craftctl default From 7411303576033a34f410d9cdf14181c922734000 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Tue, 7 Jan 2025 14:50:05 +0200 Subject: [PATCH 7/7] Revert lib --- lib/charms/postgresql_k8s/v0/postgresql.py | 195 ++++++++++----------- 1 file changed, 96 insertions(+), 99 deletions(-) diff --git a/lib/charms/postgresql_k8s/v0/postgresql.py b/lib/charms/postgresql_k8s/v0/postgresql.py index 17adae9e6..4d8d6dc30 100644 --- a/lib/charms/postgresql_k8s/v0/postgresql.py +++ b/lib/charms/postgresql_k8s/v0/postgresql.py @@ -21,11 +21,12 @@ import logging from collections import OrderedDict -from typing import Optional, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple import psycopg2 from ops.model import Relation -from psycopg2.sql import SQL, Composed, Identifier, Literal +from psycopg2 import sql +from psycopg2.sql import Composed # The unique Charmhub library identifier, never change it LIBID = "24ee217a54e840a598ff21a079c3e678" @@ -35,7 +36,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 40 +LIBPATCH = 39 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles" @@ -61,7 +62,7 @@ class PostgreSQLCreateDatabaseError(Exception): class PostgreSQLCreateUserError(Exception): """Exception raised when creating a user fails.""" - def __init__(self, message: Optional[str] = None): + def __init__(self, message: str = None): super().__init__(message) self.message = message @@ -108,14 +109,14 @@ def __init__( user: str, password: str, database: str, - system_users: Optional[list[str]] = None, + system_users: List[str] = [], ): self.primary_host = primary_host self.current_host = current_host self.user = user self.password = password self.database = database - self.system_users = system_users if system_users else [] + self.system_users = system_users def _configure_pgaudit(self, enable: bool) -> None: connection = None @@ -137,7 +138,7 @@ def _configure_pgaudit(self, enable: bool) -> None: connection.close() def _connect_to_database( - self, database: Optional[str] = None, database_host: Optional[str] = None + self, database: str = None, database_host: str = None ) -> psycopg2.extensions.connection: """Creates a connection to the database. @@ -161,8 +162,8 @@ def create_database( self, database: str, user: str, - plugins: Optional[list[str]] = None, - client_relations: Optional[list[Relation]] = None, + plugins: List[str] = [], + client_relations: List[Relation] = [], ) -> None: """Creates a new database and grant privileges to a user on it. @@ -172,25 +173,21 @@ def create_database( plugins: extensions to enable in the new database. client_relations: current established client relations. """ - plugins = plugins if plugins else [] - client_relations = client_relations if client_relations else [] try: connection = self._connect_to_database() cursor = connection.cursor() - cursor.execute( - SQL("SELECT datname FROM pg_database WHERE datname={};").format(Literal(database)) - ) + cursor.execute(f"SELECT datname FROM pg_database WHERE datname='{database}';") if cursor.fetchone() is None: - cursor.execute(SQL("CREATE DATABASE {};").format(Identifier(database))) + cursor.execute(sql.SQL("CREATE DATABASE {};").format(sql.Identifier(database))) cursor.execute( - SQL("REVOKE ALL PRIVILEGES ON DATABASE {} FROM PUBLIC;").format( - Identifier(database) + sql.SQL("REVOKE ALL PRIVILEGES ON DATABASE {} FROM PUBLIC;").format( + sql.Identifier(database) ) ) - for user_to_grant_access in [user, "admin", *self.system_users]: + for user_to_grant_access in [user, "admin"] + self.system_users: cursor.execute( - SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {};").format( - Identifier(database), Identifier(user_to_grant_access) + sql.SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {};").format( + sql.Identifier(database), sql.Identifier(user_to_grant_access) ) ) relations_accessing_this_database = 0 @@ -198,29 +195,26 @@ def create_database( for data in relation.data.values(): if data.get("database") == database: relations_accessing_this_database += 1 - with self._connect_to_database(database=database) as conn, conn.cursor() as curs: - curs.execute( - "SELECT schema_name FROM information_schema.schemata WHERE schema_name NOT LIKE 'pg_%' and schema_name <> 'information_schema';" - ) - schemas = [row[0] for row in curs.fetchall()] - statements = self._generate_database_privileges_statements( - relations_accessing_this_database, schemas, user - ) - for statement in statements: - curs.execute(statement) + with self._connect_to_database(database=database) as conn: + with conn.cursor() as curs: + curs.execute( + "SELECT schema_name FROM information_schema.schemata WHERE schema_name NOT LIKE 'pg_%' and schema_name <> 'information_schema';" + ) + schemas = [row[0] for row in curs.fetchall()] + statements = self._generate_database_privileges_statements( + relations_accessing_this_database, schemas, user + ) + for statement in statements: + curs.execute(statement) except psycopg2.Error as e: logger.error(f"Failed to create database: {e}") - raise PostgreSQLCreateDatabaseError() from e + raise PostgreSQLCreateDatabaseError() # Enable preset extensions self.enable_disable_extensions({plugin: True for plugin in plugins}, database) def create_user( - self, - user: str, - password: Optional[str] = None, - admin: bool = False, - extra_user_roles: Optional[str] = None, + self, user: str, password: str = None, admin: bool = False, extra_user_roles: str = None ) -> None: """Creates a database user. @@ -255,9 +249,7 @@ def create_user( with self._connect_to_database() as connection, connection.cursor() as cursor: # Create or update the user. - cursor.execute( - SQL("SELECT TRUE FROM pg_roles WHERE rolname={};").format(Literal(user)) - ) + cursor.execute(f"SELECT TRUE FROM pg_roles WHERE rolname='{user}';") if cursor.fetchone() is not None: user_definition = "ALTER ROLE {}" else: @@ -265,20 +257,22 @@ def create_user( user_definition += f"WITH {'NOLOGIN' if user == 'admin' else 'LOGIN'}{' SUPERUSER' if admin else ''} ENCRYPTED PASSWORD '{password}'{'IN ROLE admin CREATEDB' if admin_role else ''}" if privileges: user_definition += f" {' '.join(privileges)}" - cursor.execute(SQL("BEGIN;")) - cursor.execute(SQL("SET LOCAL log_statement = 'none';")) - cursor.execute(SQL(f"{user_definition};").format(Identifier(user))) - cursor.execute(SQL("COMMIT;")) + cursor.execute(sql.SQL("BEGIN;")) + cursor.execute(sql.SQL("SET LOCAL log_statement = 'none';")) + cursor.execute(sql.SQL(f"{user_definition};").format(sql.Identifier(user))) + cursor.execute(sql.SQL("COMMIT;")) # Add extra user roles to the new user. if roles: for role in roles: cursor.execute( - SQL("GRANT {} TO {};").format(Identifier(role), Identifier(user)) + sql.SQL("GRANT {} TO {};").format( + sql.Identifier(role), sql.Identifier(user) + ) ) except psycopg2.Error as e: logger.error(f"Failed to create user: {e}") - raise PostgreSQLCreateUserError() from e + raise PostgreSQLCreateUserError() def delete_user(self, user: str) -> None: """Deletes a database user. @@ -304,22 +298,20 @@ def delete_user(self, user: str) -> None: database ) as connection, connection.cursor() as cursor: cursor.execute( - SQL("REASSIGN OWNED BY {} TO {};").format( - Identifier(user), Identifier(self.user) + sql.SQL("REASSIGN OWNED BY {} TO {};").format( + sql.Identifier(user), sql.Identifier(self.user) ) ) - cursor.execute(SQL("DROP OWNED BY {};").format(Identifier(user))) + cursor.execute(sql.SQL("DROP OWNED BY {};").format(sql.Identifier(user))) # Delete the user. with self._connect_to_database() as connection, connection.cursor() as cursor: - cursor.execute(SQL("DROP ROLE {};").format(Identifier(user))) + cursor.execute(sql.SQL("DROP ROLE {};").format(sql.Identifier(user))) except psycopg2.Error as e: logger.error(f"Failed to delete user: {e}") - raise PostgreSQLDeleteUserError() from e + raise PostgreSQLDeleteUserError() - def enable_disable_extensions( - self, extensions: dict[str, bool], database: Optional[str] = None - ) -> None: + def enable_disable_extensions(self, extensions: Dict[str, bool], database: str = None) -> None: """Enables or disables a PostgreSQL extension. Args: @@ -361,20 +353,20 @@ def enable_disable_extensions( pass except psycopg2.errors.DependentObjectsStillExist: raise - except psycopg2.Error as e: - raise PostgreSQLEnableDisableExtensionError() from e + except psycopg2.Error: + raise PostgreSQLEnableDisableExtensionError() finally: if connection is not None: connection.close() def _generate_database_privileges_statements( - self, relations_accessing_this_database: int, schemas: list[str], user: str - ) -> list[Composed]: + self, relations_accessing_this_database: int, schemas: List[str], user: str + ) -> List[Composed]: """Generates a list of databases privileges statements.""" statements = [] if relations_accessing_this_database == 1: statements.append( - SQL( + sql.SQL( """DO $$ DECLARE r RECORD; BEGIN @@ -394,42 +386,44 @@ def _generate_database_privileges_statements( END LOOP; END; $$;""" ).format( - Identifier(user), - Identifier(user), - Identifier(user), - Identifier(user), - Identifier(user), - Identifier(user), + sql.Identifier(user), + sql.Identifier(user), + sql.Identifier(user), + sql.Identifier(user), + sql.Identifier(user), + sql.Identifier(user), ) ) statements.append( - SQL( - "UPDATE pg_catalog.pg_largeobject_metadata\n" - "SET lomowner = (SELECT oid FROM pg_roles WHERE rolname = {})\n" - "WHERE lomowner = (SELECT oid FROM pg_roles WHERE rolname = {});" - ).format(Literal(user), Literal(self.user)) + """UPDATE pg_catalog.pg_largeobject_metadata +SET lomowner = (SELECT oid FROM pg_roles WHERE rolname = '{}') +WHERE lomowner = (SELECT oid FROM pg_roles WHERE rolname = '{}');""".format(user, self.user) ) for schema in schemas: statements.append( - SQL("ALTER SCHEMA {} OWNER TO {};").format( - Identifier(schema), Identifier(user) + sql.SQL("ALTER SCHEMA {} OWNER TO {};").format( + sql.Identifier(schema), sql.Identifier(user) ) ) else: for schema in schemas: - schema = Identifier(schema) + schema = sql.Identifier(schema) statements.extend([ - SQL("GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA {} TO {};").format( - schema, Identifier(user) + sql.SQL("GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA {} TO {};").format( + schema, sql.Identifier(user) + ), + sql.SQL("GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA {} TO {};").format( + schema, sql.Identifier(user) ), - SQL("GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA {} TO {};").format( - schema, Identifier(user) + sql.SQL("GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA {} TO {};").format( + schema, sql.Identifier(user) ), - SQL("GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA {} TO {};").format( - schema, Identifier(user) + sql.SQL("GRANT USAGE ON SCHEMA {} TO {};").format( + schema, sql.Identifier(user) + ), + sql.SQL("GRANT CREATE ON SCHEMA {} TO {};").format( + schema, sql.Identifier(user) ), - SQL("GRANT USAGE ON SCHEMA {} TO {};").format(schema, Identifier(user)), - SQL("GRANT CREATE ON SCHEMA {} TO {};").format(schema, Identifier(user)), ]) return statements @@ -441,7 +435,7 @@ def get_last_archived_wal(self) -> str: return cursor.fetchone()[0] except psycopg2.Error as e: logger.error(f"Failed to get PostgreSQL last archived WAL: {e}") - raise PostgreSQLGetLastArchivedWALError() from e + raise PostgreSQLGetLastArchivedWALError() def get_current_timeline(self) -> str: """Get the timeline id for the current PostgreSQL unit.""" @@ -451,7 +445,7 @@ def get_current_timeline(self) -> str: return cursor.fetchone()[0] except psycopg2.Error as e: logger.error(f"Failed to get PostgreSQL current timeline id: {e}") - raise PostgreSQLGetCurrentTimelineError() from e + raise PostgreSQLGetCurrentTimelineError() def get_postgresql_text_search_configs(self) -> Set[str]: """Returns the PostgreSQL available text search configs. @@ -485,7 +479,10 @@ def get_postgresql_version(self, current_host=True) -> str: Returns: PostgreSQL version number. """ - host = self.current_host if current_host else None + if current_host: + host = self.current_host + else: + host = None try: with self._connect_to_database( database_host=host @@ -495,7 +492,7 @@ def get_postgresql_version(self, current_host=True) -> str: return cursor.fetchone()[0].split(" ")[1] except psycopg2.Error as e: logger.error(f"Failed to get PostgreSQL version: {e}") - raise PostgreSQLGetPostgreSQLVersionError() from e + raise PostgreSQLGetPostgreSQLVersionError() def is_tls_enabled(self, check_current_host: bool = False) -> bool: """Returns whether TLS is enabled. @@ -530,7 +527,7 @@ def list_users(self) -> Set[str]: return {username[0] for username in usernames} except psycopg2.Error as e: logger.error(f"Failed to list PostgreSQL database users: {e}") - raise PostgreSQLListUsersError() from e + raise PostgreSQLListUsersError() def list_valid_privileges_and_roles(self) -> Tuple[Set[str], Set[str]]: """Returns two sets with valid privileges and roles. @@ -561,8 +558,8 @@ def set_up_database(self) -> None: cursor.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC;") for user in self.system_users: cursor.execute( - SQL("GRANT ALL PRIVILEGES ON DATABASE postgres TO {};").format( - Identifier(user) + sql.SQL("GRANT ALL PRIVILEGES ON DATABASE postgres TO {};").format( + sql.Identifier(user) ) ) self.create_user( @@ -572,13 +569,13 @@ def set_up_database(self) -> None: cursor.execute("GRANT CONNECT ON DATABASE postgres TO admin;") except psycopg2.Error as e: logger.error(f"Failed to set up databases: {e}") - raise PostgreSQLDatabasesSetupError() from e + raise PostgreSQLDatabasesSetupError() finally: if connection is not None: connection.close() def update_user_password( - self, username: str, password: str, database_host: Optional[str] = None + self, username: str, password: str, database_host: str = None ) -> None: """Update a user password. @@ -595,17 +592,17 @@ def update_user_password( with self._connect_to_database( database_host=database_host ) as connection, connection.cursor() as cursor: - cursor.execute(SQL("BEGIN;")) - cursor.execute(SQL("SET LOCAL log_statement = 'none';")) + cursor.execute(sql.SQL("BEGIN;")) + cursor.execute(sql.SQL("SET LOCAL log_statement = 'none';")) cursor.execute( - SQL("ALTER USER {} WITH ENCRYPTED PASSWORD '" + password + "';").format( - Identifier(username) + sql.SQL("ALTER USER {} WITH ENCRYPTED PASSWORD '" + password + "';").format( + sql.Identifier(username) ) ) - cursor.execute(SQL("COMMIT;")) + cursor.execute(sql.SQL("COMMIT;")) except psycopg2.Error as e: logger.error(f"Failed to update user password: {e}") - raise PostgreSQLUpdateUserPasswordError() from e + raise PostgreSQLUpdateUserPasswordError() finally: if connection is not None: connection.close() @@ -629,8 +626,8 @@ def is_restart_pending(self) -> bool: @staticmethod def build_postgresql_parameters( - config_options: dict, available_memory: int, limit_memory: Optional[int] = None - ) -> Optional[dict]: + config_options: Dict, available_memory: int, limit_memory: Optional[int] = None + ) -> Optional[Dict]: """Builds the PostgreSQL parameters. Args: @@ -695,9 +692,9 @@ def validate_date_style(self, date_style: str) -> bool: database_host=self.current_host ) as connection, connection.cursor() as cursor: cursor.execute( - SQL( + sql.SQL( "SET DateStyle to {};", - ).format(Identifier(date_style)) + ).format(sql.Identifier(date_style)) ) return True except psycopg2.Error: