TLS integration test (#31)

* Add integration test

* Update library

* Fix PostgreSQL library

* Add relation broken test

* Add jsonschema as a binary dependency

* Change hostname to unit ip

* Add unit test dependency

* Add check for encrypted connections

* Fix space and None check

.github/workflows/ci.yaml

@@ -91,3 +91,16 @@ jobs:
           provider: lxd
       - name: Run integration tests
         run: tox -e password-rotation-integration
+
+  integration-test-lxd-tls:
+    name: Integration tests for TLS (lxd)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e tls-integration

.github/workflows/release.yaml

@@ -107,6 +107,19 @@ jobs:
       - name: Run integration tests
         run: tox -e password-rotation-integration
 
+  integration-test-tls:
+    name: Integration tests for TLS
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e tls-integration
+
   release-to-charmhub:
     name: Release to CharmHub
     needs:
@@ -118,6 +131,7 @@ jobs:
       - integration-test-db-relation
       - integration-test-db-admin-relation
       - integration-test-password-rotation
+      - integration-test-tls
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

pyproject.toml

@@ -18,6 +18,7 @@ markers = [
     "db_relation_tests",
     "db_admin_relation_tests",
     "password_rotation_tests",
+    "tls_tests",
 ]
 
 # Formatting tools configuration

tests/integration/helpers.py

@@ -14,6 +14,7 @@
 from juju.unit import Unit
 from pytest_operator.plugin import OpsTest
 from tenacity import (
+    RetryError,
     Retrying,
     retry,
     retry_if_result,
@@ -308,6 +309,7 @@ async def execute_query_on_unit(
     password: str,
     query: str,
     database: str = "postgres",
+    sslmode: str = None,
 ):
     """Execute given PostgreSQL query on a unit.
 
@@ -316,12 +318,15 @@ async def execute_query_on_unit(
         password: The PostgreSQL superuser password.
         query: Query to execute.
         database: Optional database to connect to (defaults to postgres database).
+        sslmode: Optional ssl mode to use (defaults to None).
 
     Returns:
         A list of rows that were potentially returned from the query.
     """
+    extra_connection_parameters = f"sslmode={sslmode}" if sslmode else ""
     with psycopg2.connect(
-        f"dbname='{database}' user='operator' host='{unit_address}' password='{password}' connect_timeout=10"
+        f"dbname='{database}' user='operator' host='{unit_address}'"
+        f"password='{password}' connect_timeout=10 {extra_connection_parameters}"
     ) as connection, connection.cursor() as cursor:
         cursor.execute(query)
         output = list(itertools.chain(*cursor.fetchall()))
@@ -420,6 +425,71 @@ def get_unit_address(ops_test: OpsTest, unit_name: str) -> str:
     return ops_test.model.units.get(unit_name).public_address
 
 
+async def check_tls(ops_test: OpsTest, unit_name: str, enabled: bool) -> bool:
+    """Returns whether TLS is enabled on the specific PostgreSQL instance.
+
+    Args:
+        ops_test: The ops test framework instance.
+        unit_name: The name of the unit of the PostgreSQL instance.
+        enabled: check if TLS is enabled/disabled
+
+    Returns:
+        Whether TLS is enabled/disabled.
+    """
+    unit_address = get_unit_address(ops_test, unit_name)
+    password = await get_password(ops_test, unit_name)
+    # Get the IP addresses of the other units to check that they
+    # are connecting to the primary unit (if unit_name is the
+    # primary unit name) using encrypted connections.
+    app_name = unit_name.split("/")[0]
+    unit_addresses = [
+        f"'{get_unit_address(ops_test, other_unit_name)}'"
+        for other_unit_name in ops_test.model.units
+        if other_unit_name.split("/")[0] == app_name and other_unit_name != unit_name
+    ]
+    try:
+        for attempt in Retrying(
+            stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30)
+        ):
+            with attempt:
+                output = await execute_query_on_unit(
+                    unit_address,
+                    password,
+                    "SHOW ssl;",
+                    sslmode="require" if enabled else "disable",
+                )
+                tls_enabled = "on" in output
+
+                # Check for the number of bits in the encryption algorithm used
+                # on each connection. If a connection is not encrypted, None
+                # is returned instead of an integer.
+                connections_encryption_info = await execute_query_on_unit(
+                    unit_address,
+                    password,
+                    "SELECT bits FROM pg_stat_ssl INNER JOIN pg_stat_activity"
+                    " ON pg_stat_ssl.pid = pg_stat_activity.pid"
+                    " WHERE pg_stat_ssl.pid = pg_backend_pid()"
+                    f" OR client_addr IN ({','.join(unit_addresses)});",
+                )
+
+                # This flag indicates whether all the connections are encrypted
+                # when checking for TLS enabled or all the connections are not
+                # encrypted when checking for TLS disabled.
+                connections_encrypted = (
+                    all(connections_encryption_info)
+                    if enabled
+                    else any(connections_encryption_info)
+                )
+
+                if enabled != tls_enabled or tls_enabled != connections_encrypted:
+                    raise ValueError(
+                        f"TLS is{' not' if not tls_enabled else ''} enabled on {unit_name}"
+                    )
+                return True
+    except RetryError:
+        return False
+
+
 async def scale_application(ops_test: OpsTest, application_name: str, count: int) -> None:
     """Scale a given application to a specific unit count.
 

tests/integration/test_tls.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+import pytest as pytest
+from pytest_operator.plugin import OpsTest
+
+from tests.helpers import METADATA
+from tests.integration.helpers import DATABASE_APP_NAME, check_tls
+
+APP_NAME = METADATA["name"]
+TLS_CERTIFICATES_APP_NAME = "tls-certificates-operator"
+
+
+@pytest.mark.abort_on_fail
+@pytest.mark.tls_tests
+@pytest.mark.skip_if_deployed
+async def test_deploy_active(ops_test: OpsTest):
+    """Build the charm and deploy it."""
+    charm = await ops_test.build_charm(".")
+    async with ops_test.fast_forward():
+        await ops_test.model.deploy(
+            charm, resources={"patroni": "patroni.tar.gz"}, application_name=APP_NAME, num_units=3
+        )
+        await ops_test.juju("attach-resource", APP_NAME, "patroni=patroni.tar.gz")
+        await ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", timeout=1000)
+
+
+@pytest.mark.tls_tests
+async def test_tls_enabled(ops_test: OpsTest) -> None:
+    """Test that TLS is enabled when relating to the TLS Certificates Operator."""
+    async with ops_test.fast_forward():
+        # Deploy TLS Certificates operator.
+        config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"}
+        await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="edge", config=config)
+        await ops_test.model.wait_for_idle(
+            apps=[TLS_CERTIFICATES_APP_NAME], status="active", timeout=1000
+        )
+
+        # Relate it to the PostgreSQL to enable TLS.
+        await ops_test.model.relate(DATABASE_APP_NAME, TLS_CERTIFICATES_APP_NAME)
+        await ops_test.model.wait_for_idle(status="active", timeout=1000)
+
+        # Wait for all units enabling TLS.
+        for unit in ops_test.model.applications[DATABASE_APP_NAME].units:
+            assert await check_tls(ops_test, unit.name, enabled=True)
+
+        # Remove the relation.
+        await ops_test.model.applications[DATABASE_APP_NAME].remove_relation(
+            f"{DATABASE_APP_NAME}:certificates", f"{TLS_CERTIFICATES_APP_NAME}:certificates"
+        )
+        await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1000)
+
+        # Wait for all units disabling TLS.
+        for unit in ops_test.model.applications[DATABASE_APP_NAME].units:
+            assert await check_tls(ops_test, unit.name, enabled=False)

tox.ini

@@ -161,6 +161,25 @@ commands =
 whitelist_externals =
     sh
 
+[testenv:tls-integration]
+description = Run TLS integration tests
+deps =
+    pytest
+    juju==2.9.11 # juju 3.0.0 has issues with retrieving action results
+    landscape-api-py3
+    mailmanclient
+    pytest-operator
+    psycopg2-binary
+    -r{toxinidir}/requirements.txt
+commands =
+    # Download patroni resource to use in the charm deployment.
+    sh -c 'stat patroni.tar.gz > /dev/null 2>&1 || curl "https://github.com/zalando/patroni/archive/refs/tags/v2.1.3.tar.gz" -L -s > patroni.tar.gz'
+    pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m tls_tests
+    # Remove the downloaded resource.
+    sh -c 'rm -f patroni.tar.gz'
+whitelist_externals =
+    sh
+
 [testenv:integration]
 description = Run all integration tests
 deps =