Merge remote-tracking branch 'origin/6/edge' into gatici-changes

.github/workflows/ci.yaml

@@ -12,17 +12,6 @@ on:
     - cron: "53 0 * * *" # Daily at 00:53 UTC
   # Triggered on push to branch "main" by .github/workflows/release.yaml
   workflow_call:
-    secrets:
-      CHARMHUB_TOKEN:
-        required: true
-      AWS_ACCESS_KEY:
-        required: true
-      AWS_SECRET_KEY:
-        required: true
-      GCP_ACCESS_KEY:
-        required: true
-      GCP_SECRET_KEY:
-        required: true
 
 jobs:
   lint:
@@ -98,73 +87,37 @@ jobs:
           exit "$FMT_STATUS"
 
   build:
-    name: Build charms
-    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v8
-
-  integration-test:
     strategy:
-      fail-fast: false
       matrix:
-        tox-environments:
-          - charm-integration
-          - ha-integration
-          - relation-integration
-          - legacy-integration
-          - tls-integration
-          - backup-integration
-          - metric-integration
-          - sharding-integration
-          - sharding-relation-integration
-          - sharding-race-conditions-integration
-          - sharding-mongos-integration
-    name: ${{ matrix.tox-environments }}
+        path:
+          - .
+          - tests/integration/sharding_tests/application
+          - tests/integration/relation_tests/new_relations/application-charm
+          - tests/integration/dummy_legacy_app
+    name: Build charm
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    with:
+      path-to-charm-directory: ${{ matrix.path }}
+      cache: true
+
+  integration-test:
+    name: Integration test charm
     needs:
       - lint
       - unit-test
-      - lib-check
       - build
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        # TODO: Replace with custom image on self-hosted runner
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-          juju-channel: 3.1/stable
-      - name: Download packed charm(s)
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ needs.build.outputs.artifact-name }}
-      - name: Free disk space
-        run: |
-          echo "Free disk space before cleanup"
-          df -T
-          # free space in the runner
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /usr/local/share/boost
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          echo "Free disk space after cleanup"
-          df -T
-      - name: Select tests
-        id: select-tests
-        run: |
-          if [ "${{ github.event_name }}" == "schedule" ]
-          then
-            echo Running unstable and stable tests
-            echo "mark_expression=" >> $GITHUB_OUTPUT
-          else
-            echo Skipping unstable tests
-            echo "mark_expression=not unstable" >> $GITHUB_OUTPUT
-          fi
-      - name: Run integration tests
-        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}'
-        env:
-          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
-          AWS_ACCESS_KEY: ${{ matrix.tox-environments != 'backup-integration' || secrets.AWS_ACCESS_KEY }}
-          AWS_SECRET_KEY: ${{ matrix.tox-environments != 'backup-integration' || secrets.AWS_SECRET_KEY }}
-          GCP_ACCESS_KEY: ${{ matrix.tox-environments != 'backup-integration' || secrets.GCP_ACCESS_KEY }}
-          GCP_SECRET_KEY: ${{ matrix.tox-environments != 'backup-integration' || secrets.GCP_SECRET_KEY }}
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    with:
+      artifact-prefix: packed-charm-cache-true
+      cloud: lxd
+      juju-agent-version: 3.1.6
+    permissions:
+      contents: write # Needed for Allure Report beta
+    secrets:
+      integration-test: |
+        {
+          "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",
+          "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}",
+          "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}",
+          "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}",
+        }

.github/workflows/release.yaml

@@ -8,25 +8,18 @@ on:
 jobs:
   ci-tests:
     uses: ./.github/workflows/ci.yaml
-    secrets:
-      CHARMHUB_TOKEN: "${{ secrets.CHARMHUB_TOKEN }}"
-      AWS_ACCESS_KEY: "${{ secrets.AWS_ACCESS_KEY }}"
-      AWS_SECRET_KEY: "${{ secrets.AWS_SECRET_KEY }}"
-      GCP_ACCESS_KEY: "${{ secrets.GCP_ACCESS_KEY }}"
-      GCP_SECRET_KEY: "${{ secrets.GCP_SECRET_KEY }}"
+    secrets: inherit
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm_without_cache.yaml@v8
-    with:
-      charmcraft-snap-channel: "latest/stable"
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
 
   release-charm:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v8
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v11.0.1
     with:
       channel: 6/edge
       artifact-name: ${{ needs.build.outputs.artifact-name }}
@@ -36,7 +29,7 @@ jobs:
       contents: write # Needed to create GitHub release
 
   release-libraries:
-    name: Release libraries 
+    name: Release libraries
     runs-on: ubuntu-latest
     needs:
       - ci-tests

lib/charms/mongodb/v0/config_server_interface.py

@@ -35,7 +35,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 6
+LIBPATCH = 7
 
 
 class ClusterProvider(Object):
@@ -275,7 +275,13 @@ def _on_relation_broken(self, event: RelationBrokenEvent) -> None:
 
     def is_mongos_running(self) -> bool:
         """Returns true if mongos service is running."""
-        with MongosConnection(None, f"mongodb://{MONGOS_SOCKET_URI_FMT}") as mongo:
+        connection_uri = f"mongodb://{self.charm.get_mongos_host()}"
+
+        # when running internally, connections through Unix Domain sockets do not need port.
+        if self.charm.is_external_client:
+            connection_uri = connection_uri + f":{Config.MONGOS_PORT}"
+
+        with MongosConnection(None, connection_uri) as mongo:
             return mongo.is_ready
 
     def update_config_server_db(self, config_server_db) -> bool:
@@ -285,7 +291,10 @@ def update_config_server_db(self, config_server_db) -> bool:
 
         mongos_config = self.charm.mongos_config
         mongos_start_args = get_mongos_args(
-            mongos_config, snap_install=True, config_server_db=config_server_db
+            mongos_config,
+            snap_install=True,
+            config_server_db=config_server_db,
+            external_connectivity=self.charm.is_external_client,
         )
         add_args_to_env("MONGOS_ARGS", mongos_start_args)
         return True

lib/charms/mongodb/v0/mongodb.py

@@ -22,6 +22,8 @@
     wait_fixed,
 )
 
+from config import Config
+
 # The unique Charmhub library identifier, never change it
 LIBID = "49c69d9977574dd7942eb7b54f43355b"
 
@@ -30,7 +32,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 7
+LIBPATCH = 8
 
 # path to store mongodb ketFile
 logger = logging.getLogger(__name__)
@@ -57,6 +59,7 @@ class MongoDBConfiguration:
     roles: Set[str]
     tls_external: bool
     tls_internal: bool
+    standalone: bool = False
 
     @property
     def uri(self):
@@ -66,6 +69,14 @@ def uri(self):
         auth_source = ""
         if self.database != "admin":
             auth_source = "&authSource=admin"
+
+        if self.standalone:
+            return (
+                f"mongodb://{quote_plus(self.username)}:"
+                f"{quote_plus(self.password)}@"
+                f"localhost:{Config.MONGODB_PORT}/?authSource=admin"
+            )
+
         return (
             f"mongodb://{quote_plus(self.username)}:"
             f"{quote_plus(self.password)}@"

lib/charms/mongodb/v1/helpers.py

@@ -30,7 +30,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 3
+LIBPATCH = 4
 
 # path to store mongodb ketFile
 KEY_FILE = "keyFile"
@@ -96,6 +96,7 @@ def get_mongos_args(
     config,
     snap_install: bool = False,
     config_server_db: str = None,
+    external_connectivity: bool = True,
 ) -> str:
     """Returns the arguments used for starting mongos on a config-server side application.
 
@@ -104,9 +105,9 @@ def get_mongos_args(
     """
     # suborinate charm which provides its own config_server_db, should only use unix domain socket
     binding_ips = (
-        f"--bind_ip {MONGODB_COMMON_DIR}/var/mongodb-27018.sock"
-        if config_server_db
-        else "--bind_ip_all"
+        "--bind_ip_all"
+        if external_connectivity
+        else f"--bind_ip {MONGODB_COMMON_DIR}/var/mongodb-27018.sock"
     )
 
     # mongos running on the config server communicates through localhost

lib/charms/mongodb/v1/mongodb_backups.py

@@ -40,7 +40,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 2
+LIBPATCH = 4
 
 logger = logging.getLogger(__name__)
 
@@ -57,7 +57,7 @@
 REMAPPING_PATTERN = r"\ABackup doesn't match current cluster topology - it has different replica set names. Extra shards in the backup will cause this, for a simple example. The extra/unknown replica set names found in the backup are: ([^,\s]+)([.] Backup has no data for the config server or sole replicaset)?\Z"
 PBM_STATUS_CMD = ["status", "-o", "json"]
 MONGODB_SNAP_DATA_DIR = "/var/snap/charmed-mongodb/current"
-BACKUP_RESTORE_MAX_ATTEMPTS = 5
+BACKUP_RESTORE_MAX_ATTEMPTS = 10
 BACKUP_RESTORE_ATTEMPT_COOLDOWN = 15
 
 
@@ -112,18 +112,34 @@ def __init__(self, charm):
 
         # s3 relation handles the config options for s3 backups
         self.s3_client = S3Requirer(self.charm, S3_RELATION)
+        self.framework.observe(
+            self.charm.on[S3_RELATION].relation_joined, self.on_s3_relation_joined
+        )
         self.framework.observe(
             self.s3_client.on.credentials_changed, self._on_s3_credential_changed
         )
         self.framework.observe(self.charm.on.create_backup_action, self._on_create_backup_action)
         self.framework.observe(self.charm.on.list_backups_action, self._on_list_backups_action)
         self.framework.observe(self.charm.on.restore_action, self._on_restore_action)
 
+    def on_s3_relation_joined(self, _) -> None:
+        """Checks for valid integration for s3-integrations."""
+        if not self.is_valid_s3_integration():
+            logger.debug(
+                "Shard does not support s3 relations, please relate s3-integrator to config-server only."
+            )
+            self.charm.unit.status = BlockedStatus(
+                "Relation to s3-integrator is not supported, config role must be config-server"
+            )
+
     def _on_s3_credential_changed(self, event: CredentialsChangedEvent):
         """Sets pbm credentials, resyncs if necessary and reports config errors."""
         # handling PBM configurations requires that MongoDB is running and the pbm snap is
         # installed.
         action = "configure-pbm"
+        if not self._pass_sanity_checks(event, action):
+            return
+
         if not self.charm.db_initialised:
             self._defer_action_with_info_log(
                 event, action, "Set PBM credentials, MongoDB not ready."
@@ -140,12 +156,7 @@ def _on_s3_credential_changed(self, event: CredentialsChangedEvent):
 
     def _on_create_backup_action(self, event) -> None:
         action = "backup"
-        if self.model.get_relation(S3_RELATION) is None:
-            self._fail_action_with_error_log(
-                event,
-                action,
-                "Relation with s3-integrator charm missing, cannot create backup.",
-            )
+        if not self._pass_sanity_checks(event, action):
             return
 
         # only leader can create backups. This prevents multiple backups from being attempted at
@@ -195,12 +206,7 @@ def _on_create_backup_action(self, event) -> None:
 
     def _on_list_backups_action(self, event) -> None:
         action = "list-backups"
-        if self.model.get_relation(S3_RELATION) is None:
-            self._fail_action_with_error_log(
-                event,
-                action,
-                "Relation with s3-integrator charm missing, cannot list backups.",
-            )
+        if not self._pass_sanity_checks(event, action):
             return
 
         # cannot list backups if pbm is resyncing, or has incompatible options or incorrect
@@ -229,12 +235,7 @@ def _on_list_backups_action(self, event) -> None:
 
     def _on_restore_action(self, event) -> None:
         action = "restore"
-        if self.model.get_relation(S3_RELATION) is None:
-            self._fail_action_with_error_log(
-                event,
-                action,
-                "Relation with s3-integrator charm missing, cannot restore from a backup.",
-            )
+        if not self._pass_sanity_checks(event, action):
             return
 
         backup_id = event.params.get("backup-id")
@@ -289,6 +290,38 @@ def _on_restore_action(self, event) -> None:
             self._fail_action_with_error_log(event, action, str(restore_error))
 
     # BEGIN: helper functions
+    def is_valid_s3_integration(self) -> bool:
+        """Return true if relation to s3-integrator is valid.
+
+        Only replica sets and config servers can integrate to s3-integrator.
+        """
+        if self.charm.is_role(Config.Role.SHARD) and self.model.get_relation(S3_RELATION):
+            return False
+
+        return True
+
+    def _pass_sanity_checks(self, event, action) -> bool:
+        """Return True if basic pre-conditions for running backup actions are met.
+
+        No matter what backup-action is being run, these requirements must be met.
+        """
+        if not self.is_valid_s3_integration():
+            self._fail_action_with_error_log(
+                event,
+                action,
+                "Shards do not support backup operations, please run action on config-server.",
+            )
+            return False
+
+        if self.model.get_relation(S3_RELATION) is None:
+            self._fail_action_with_error_log(
+                event,
+                action,
+                "Relation with s3-integrator charm missing, cannot restore from a backup.",
+            )
+            return False
+
+        return True
 
     def _configure_pbm_options(self, event) -> None:
         action = "configure-pbm"