Cache charmcraft pack container, skip unstable tests except on schedu…

…le (#168)

Ported from canonical/mysql-k8s-operator#146

.github/workflows/ci.yaml

@@ -1,6 +1,12 @@
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
 name: Tests
+
 on:
   pull_request:
+  schedule:
+    - cron: '53 0 * * *' # Daily at 00:53 UTC
+  # Triggered on push to branch "main" by .github/workflows/release.yaml
   workflow_call:
     secrets:
       CHARMHUB_TOKEN:
@@ -21,21 +27,23 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
         run: python3 -m pip install tox
       - name: Run linters
-        run: tox -e lint
+        run: tox run -e lint
 
   unit-test:
     name: Unit tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
-        run: python -m pip install tox
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
+        run: python3 -m pip install tox
       - name: Run tests
-        run: tox -e unit
+        run: tox run -e unit
 
   lib-check:
     name: Check libraries
@@ -51,109 +59,56 @@ jobs:
           credentials: "${{ secrets.CHARMHUB_TOKEN }}" # FIXME: current token will expire in 2023-07-04
           github-token: "${{ secrets.GITHUB_TOKEN }}"
 
-  charm-integration-test-lxd:
-    name: Charm integration tests (lxd)
-    needs:
-      - lint
-      - unit-test
-      - lib-check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-      - name: Run charm integration tests
-        run: tox -e charm-integration
+  build:
+    name: Build charms
+    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1
 
-  ha-integration-test-lxd:
-    name: HA integration tests (lxd)
+  integration-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        tox-environments:
+          - charm-integration
+          - ha-integration
+          - relation-integration
+          - legacy-integration
+          - tls-integration
+          - backup-integration
+    name: ${{ matrix.tox-environments }}
     needs:
       - lint
       - unit-test
       - lib-check
+      - build
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Setup operator environment
+        # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: lxd
-      - name: Run ha integration tests
-        run: tox -e ha-integration
-
-  relation-integration-test-lxd:
-    name: Relation integration tests (lxd)
-    needs:
-      - lint
-      - unit-test
-      - lib-check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
+      - name: Download packed charm(s)
+        uses: actions/download-artifact@v3
         with:
-          provider: lxd
-      - name: Run relation integration tests
-        run: tox -e relation-integration
-
-  legacy-relation-integration-test-lxd:
-    name: Legacay relation integration tests (lxd)
-    needs:
-      - lint
-      - unit-test
-      - lib-check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-      - name: Run legacy relation integration tests
-        run: tox -e legacy-integration
-
-  tls-integration-test-lxd:
-    name: TLS integration tests (lxd)
-    needs:
-      - lint
-      - unit-test
-      - lib-check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-      - name: Run tls integration tests
-        run: tox -e tls-integration
-
-  backup-integration-test-lxd:
-    name: Backup integration tests (lxd)
-    needs:
-      - lint
-      - unit-test
-      - lib-check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-      - name: Run backup integration tests
-        run: tox -e backup-integration
+          name: ${{ needs.build.outputs.artifact-name }}
+      - name: Select tests
+        id: select-tests
+        run: |
+          if [ "${{ github.event_name }}" == "schedule" ]
+          then
+            echo Running unstable and stable tests
+            echo "mark_expression=" >> $GITHUB_OUTPUT
+          else
+            echo Skipping unstable tests
+            echo "mark_expression=not unstable" >> $GITHUB_OUTPUT
+          fi
+      - name: Run integration tests
+        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}'
         env:
-          AWS_ACCESS_KEY: "${{ secrets.AWS_ACCESS_KEY }}"
-          AWS_SECRET_KEY: "${{ secrets.AWS_SECRET_KEY }}"
-          GCP_ACCESS_KEY: "${{ secrets.GCP_ACCESS_KEY }}"
-          GCP_SECRET_KEY: "${{ secrets.GCP_SECRET_KEY }}"
+          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
+          AWS_ACCESS_KEY: ${{ matrix.tox-environments != 'backup-integration' || secrets.AWS_ACCESS_KEY }}
+          AWS_SECRET_KEY: ${{ matrix.tox-environments != 'backup-integration' || secrets.AWS_SECRET_KEY }}
+          GCP_ACCESS_KEY: ${{ matrix.tox-environments != 'backup-integration' || secrets.GCP_ACCESS_KEY }}
+          GCP_SECRET_KEY: ${{ matrix.tox-environments != 'backup-integration' || secrets.GCP_SECRET_KEY }}

CONTRIBUTING.md

@@ -55,26 +55,26 @@ pip install -r requirements.txt
 ## Testing
 
 ```shell
-tox -e fmt                      # update your code according to linting rules
-tox -e lint                     # code style
-tox -e unit                     # unit tests
-tox -e integration              # run all integration tests
-tox -e charm-integration        # charm integration tests
-tox -e ha-integration           # high-availability replication integration tests
-tox -e relation-integration     # relation integration tests (legacy and new relations)
-tox                             # runs 'lint' and 'unit' environments
+tox run -e format                   # update your code according to linting rules
+tox run -e lint                     # code style
+tox run -e unit                     # unit tests
+tox run -e integration              # run all integration tests
+tox run -e charm-integration        # charm integration tests
+tox run -e ha-integration           # high-availability replication integration tests
+tox run -e relation-integration     # relation integration tests (legacy and new relations)
+tox                                 # runs 'lint' and 'unit' environments
 ```
 
 Testing high availability on a production cluster can be done with:
 ```shell
-tox -e ha-integration -- --model=<model_name>
+tox run -e ha-integration -- --model=<model_name>
 ```
 
-Note if you'd like to test storage re-use in ha-testing, your storage must not be of the type `rootfs`. `rootfs` storage is tied to the machine lifecycle and does not stick around after unit removal. `rootfs` storage is used by default with `tox -e ha-integration`. To test ha-testing for storage re-use: 
+Note if you'd like to test storage re-use in ha-testing, your storage must not be of the type `rootfs`. `rootfs` storage is tied to the machine lifecycle and does not stick around after unit removal. `rootfs` storage is used by default with `tox run -e ha-integration`. To test ha-testing for storage re-use: 
 ```shell
 juju create-storage-pool mongodb-ebs ebs volume-type=standard # create a storage pool
 juju deploy ./*charm --storage mongodb=mongodb-ebs,7G,1 # deploy 1 or more units of application with said storage pool
-tox -e ha-integration -- --model=default # run tests in the model where you deployed the app 
+tox run -e ha-integration -- --model=default # run tests in the model where you deployed the app 
 ```
 
 ## Build Charm

pyproject.toml

@@ -12,6 +12,7 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
+markers = ["unstable"]
 
 # Formatting tools configuration
 [tool.black]

tests/integration/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/backup_tests/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/backup_tests/helpers.py

@@ -7,7 +7,7 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import RetryError, Retrying, stop_after_attempt, wait_fixed
 
-import tests.integration.ha_tests.helpers as ha_helpers
+from ..ha_tests import helpers as ha_helpers
 
 S3_APP_NAME = "s3-integrator"
 TIMEOUT = 10 * 60

tests/integration/backup_tests/test_backups.py

@@ -6,12 +6,12 @@
 import string
 import time
 
-import helpers
 import pytest
 from pytest_operator.plugin import OpsTest
 from tenacity import RetryError, Retrying, stop_after_delay, wait_fixed
 
-import tests.integration.ha_tests.helpers as ha_helpers
+from ..ha_tests import helpers as ha_helpers
+from . import helpers
 
 S3_APP_NAME = "s3-integrator"
 TIMEOUT = 15 * 60

tests/integration/conftest.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from pytest_operator.plugin import OpsTest
+
+
+@pytest.fixture(scope="module")
+def ops_test(ops_test: OpsTest) -> OpsTest:
+    if os.environ.get("CI") == "true":
+        # Running in GitHub Actions; skip build step
+        # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml)
+        packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"])
+
+        async def build_charm(charm_path, bases_index: int = None) -> Path:
+            for charm in packed_charms:
+                if Path(charm_path) == Path(charm["directory_path"]):
+                    if bases_index is None or bases_index == charm["bases_index"]:
+                        return charm["file_path"]
+            raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}")
+
+        ops_test.build_charm = build_charm
+    return ops_test

tests/integration/ha_tests/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/ha_tests/test_ha.py

@@ -6,12 +6,13 @@
 import asyncio
 import time
 
-import helpers
 import pytest
 from pymongo import MongoClient
 from pytest_operator.plugin import OpsTest
 from tenacity import RetryError, Retrying, stop_after_delay, wait_fixed
 
+from . import helpers
+
 ANOTHER_DATABASE_APP_NAME = "another-database-a"
 MONGOD_PROCESS = "/usr/bin/mongod"
 MEDIAN_REELECTION_TIME = 12

tests/integration/relation_tests/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/relation_tests/legacy_relations/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/relation_tests/legacy_relations/helpers.py

@@ -10,7 +10,7 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import RetryError, Retrying, stop_after_delay, wait_fixed
 
-from tests.integration.relation_tests.legacy_relations.api import GraylogApi
+from .api import GraylogApi
 
 logger = logging.getLogger(__name__)
 DEFAULT_REST_API_TIMEOUT = 120

tests/integration/relation_tests/legacy_relations/test_charm_legacy_relations.py

@@ -8,7 +8,7 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import RetryError, Retrying, stop_after_delay, wait_fixed
 
-from tests.integration.relation_tests.legacy_relations.helpers import (
+from .helpers import (
     GRAYLOG_APP_NAME,
     ApiTimeoutError,
     _verify_rest_api_is_alive,

tests/integration/relation_tests/new_relations/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/relation_tests/new_relations/test_charm_relations.py

@@ -12,11 +12,8 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import RetryError
 
-from tests.integration.ha_tests.helpers import replica_set_primary
-from tests.integration.relation_tests.new_relations.helpers import (
-    get_application_relation_data,
-    verify_application_data,
-)
+from ...ha_tests.helpers import replica_set_primary
+from .helpers import get_application_relation_data, verify_application_data
 
 MEDIAN_REELECTION_TIME = 12
 APPLICATION_APP_NAME = "application"

tests/integration/test_charm.py

@@ -7,7 +7,13 @@
 import time
 
 import pytest
-from helpers import (
+from pymongo import MongoClient
+from pymongo.errors import PyMongoError, ServerSelectionTimeoutError
+from pytest_operator.plugin import OpsTest
+from tenacity import RetryError
+
+from .ha_tests.helpers import app_name, kill_unit_process
+from .helpers import (
     APP_NAME,
     PORT,
     UNIT_IDS,
@@ -16,12 +22,6 @@
     get_password,
     unit_uri,
 )
-from pymongo import MongoClient
-from pymongo.errors import PyMongoError, ServerSelectionTimeoutError
-from pytest_operator.plugin import OpsTest
-from tenacity import RetryError
-
-from tests.integration.ha_tests.helpers import app_name, kill_unit_process
 
 logger = logging.getLogger(__name__)
 

tests/integration/tls_tests/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/tls_tests/helpers.py

@@ -7,8 +7,8 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import RetryError, Retrying, stop_after_attempt, wait_exponential
 
-from tests.integration.ha_tests.helpers import app_name
-from tests.integration.helpers import get_password
+from ..ha_tests.helpers import app_name
+from ..helpers import get_password
 
 PORT = 27017