Move to DP workflows and self-hosted runners (#440)

## Issue
This PR: 
- moves our CI pipelines to using the DP workflows 
    - moves to poetry
    - enables allure reports 
- makes integ. tests run on self-hosted runners
- bumps juju version to `3.4.4` on CI
- updates the dp libs on client integration tests

.github/workflows/ci.yaml

@@ -16,18 +16,19 @@ on:
 jobs:
   lint:
     name: Lint
-    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v16.7.0
 
   unit-test:
-    name: Unit tests
+    name: Unit test charm
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 10
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install tox
-        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
-        run: python3 -m pip install --upgrade pip; python3 -m pip install tox
+      - name: Install tox & poetry
+        run: |
+          pipx install tox
+          pipx install poetry
       - name: Run tests
         run: tox run -e unit
 
@@ -40,11 +41,16 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      - run: |
+          # Workaround for https://github.com/canonical/charmcraft/issues/1389#issuecomment-1880921728
+          touch requirements.txt
       - name: Check libs
         uses: canonical/charming-actions/[email protected]
         with:
-          credentials: "${{ secrets.CHARMHUB_TOKEN }}" # FIXME: current token will expire in 2023-07-04
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
+          credentials: ${{ secrets.CHARMHUB_TOKEN }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          use-labels: false
+          fail-build: ${{ github.event_name == 'pull_request' }}
 
   check-terraform:
     name: Check Terraform
@@ -78,29 +84,31 @@ jobs:
 
   build:
     strategy:
+      fail-fast: false
       matrix:
         path:
           - .
           - tests/integration/sharding_tests/application
           - tests/integration/relation_tests/new_relations/application-charm
           - tests/integration/dummy_legacy_app
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v16.7.0
     with:
       path-to-charm-directory: ${{ matrix.path }}
       cache: true
 
   integration-test:
-    name: Integration test charm
+    name: Integration test charm | 3.4.4
     needs:
       - lint
       - unit-test
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v16.7.0
     with:
       artifact-prefix: packed-charm-cache-true
       cloud: lxd
-      juju-agent-version: 3.1.7
+      juju-agent-version: 3.4.4
+      _beta_allure_report: true
     permissions:
       contents: write # Needed for Allure Report beta
     secrets:

.github/workflows/release.yaml

@@ -9,17 +9,19 @@ jobs:
   ci-tests:
     uses: ./.github/workflows/ci.yaml
     secrets: inherit
+    permissions:
+      contents: write  # Needed for Allure Report beta
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v16.7.0
 
   release-charm:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v16.7.0
     with:
       channel: 6/edge
       artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}

.github/workflows/sync_issue_to_jira.yaml

@@ -9,7 +9,7 @@ on:
 jobs:
   sync:
     name: Sync GitHub issue to Jira
-    uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v16.7.0
     with:
       jira-base-url: https://warthogs.atlassian.net
       jira-project-key: DPE

.gitignore

@@ -1,3 +1,6 @@
+/requirements.txt
+/requirements-last-build.txt
+
 *.idea
 .vscode/
 .coverage

CONTRIBUTING.md

@@ -36,6 +36,19 @@ lxc network set lxdbr0 ipv6.address none
 # bootstrap controller to lxd
 juju clouds
 juju bootstrap localhost overlord
+
+# Install environment dependencies
+sudo apt-get install python3-pip python3-venv -y --no-install-recommends
+python3 -m pip install pipx
+python3 -m pipx ensurepath
+          
+pipx install tox
+pipx install poetry
+pipx inject poetry poetry-plugin-export
+# TODO: Remove after https://github.com/python-poetry/poetry/pull/5980 is closed
+poetry config warnings.export false
+
+pipx install charmcraftcache
 ```
 
 Clone this repository:
@@ -48,7 +61,6 @@ Create and activate a virtualenv, and install the development requirements:
 ```shell
 virtualenv -p python3 venv
 source venv/bin/activate
-pip install -r requirements.txt
 ```
 
 
@@ -58,31 +70,34 @@ pip install -r requirements.txt
 tox run -e format                   # update your code according to linting rules
 tox run -e lint                     # code style
 tox run -e unit                     # unit tests
-tox run -e integration              # run all integration tests
-tox run -e charm-integration        # charm integration tests
-tox run -e ha-integration           # high-availability replication integration tests
-tox run -e relation-integration     # relation integration tests (legacy and new relations)
-tox                                 # runs 'lint' and 'unit' environments
+
+tox run -e integration --group='1' -m 'not unstable'                      # run all integration tests
+tox run -e integration -- 'tests/integration/test_charm.py' --group='1'   # charm integration tests
+...
+
+# In general, to run any integration test:
+tox run -e integration -- 'tests/integration/<path_to_test_module>.py' --group='1' # runs <test_module> tests
 ```
 
 Testing high availability on a production cluster can be done with:
 ```shell
-tox run -e ha-integration -- --model=<model_name>
+tox run -e integration -- 'tests/integration/ha_tests/test_ha.py' --group='1' --model=<model_name>
 ```
 
 Note if you'd like to test storage reuse in ha-testing, your storage must not be of the type `rootfs`. `rootfs` storage is tied to the machine lifecycle and does not stick around after unit removal. `rootfs` storage is used by default with `tox run -e ha-integration`. To test ha-testing for storage reuse: 
 ```shell
+juju add-model test
 juju create-storage-pool mongodb-ebs ebs volume-type=standard # create a storage pool
 juju deploy ./*charm --storage mongodb=mongodb-ebs,7G,1 # deploy 1 or more units of application with said storage pool
-tox run -e ha-integration -- --model=default # run tests in the model where you deployed the app 
+tox run -e integration -- 'tests/integration/ha_tests/test_ha.py' --group='1' --model=test # run tests in the model where you deployed the app 
 ```
 
 ## Build Charm
 
 Build the charm in this git repository using:
 
 ```shell
-charmcraft pack
+tox run -e build-dev -- -v --bases-index='0'
 ```
 
 ## Deploy Charm

charm_internal_version

@@ -1 +1 @@
-6
+7

charmcraft.yaml

@@ -11,13 +11,19 @@ bases:
         channel: "22.04"
 parts:
   charm:
+    override-build: |
+      rustup default stable
+      craftctl default
+    build-snaps:
+      - rustup
     charm-binary-python-packages:
       - setuptools
       - cryptography
     build-packages:
-      - git
+      - build-essential
       - libffi-dev
       - libssl-dev
+      - pkg-config
       - rustc
       - cargo
     prime: