patch(integration_test_charm.yaml): Add Allure "unknown" status for tests with missing results

.github/workflows/integration_test_charm.yaml

@@ -150,11 +150,33 @@ jobs:
               print("Skipping unstable tests")
               output = "mark_expression=not unstable"
 
+          with open(os.environ["GITHUB_OUTPUT"], "a") as file:
+              file.write(output)
+      - name: (beta) Get Allure collection option
+        if: ${{ inputs._beta_allure_report }}
+        id: allure-collection-option
+        # TODO future improvement: check if allure-pytest installed instead
+        shell: python
+        run: |
+          import os
+
+          output = "option=--allure-collection-dir=allure-collection-default-results"
+
           with open(os.environ["GITHUB_OUTPUT"], "a") as file:
               file.write(output)
       - name: Collect test groups
         id: collect-groups
-        run: tox run -e integration -- tests/integration -m '${{ steps.select-test-stability.outputs.mark_expression }}' --collect-groups
+        run: tox run -e integration -- tests/integration -m '${{ steps.select-test-stability.outputs.mark_expression }}' --collect-groups ${{ steps.allure-collection-option.outputs.option }}
+      - name: (beta) Upload Allure collection results
+        # Default test results in case the integration tests time out or runner set up fails
+        # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test)
+        if: ${{ inputs._beta_allure_report && github.event_name == 'schedule' && github.run_attempt == '1' }}
+        uses: actions/upload-artifact@v4
+        with:
+          # TODO future improvement: ensure artifact name is unique (if called with a matrix that changes inputs)
+          name: allure-collection-default-results-integration-test-charm
+          path: allure-collection-default-results/
+          if-no-files-found: error
     outputs:
       groups: ${{ steps.collect-groups.outputs.groups }}
       default_runner: ${{ steps.parse-architecture.outputs.default_runner }}
@@ -399,6 +421,15 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
+      - name: Get workflow version
+        id: workflow-version
+        uses: canonical/get-workflow-version-action@v1
+        with:
+          repository-name: canonical/data-platform-workflows
+          file-name: integration_test_charm.yaml
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install CLI
+        run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli
       - name: Download Allure
         # Following instructions from https://allurereport.org/docs/gettingstarted-installation/#install-via-the-system-package-manager-for-linux
         run: gh release download --repo allure-framework/allure2 --pattern 'allure_*.deb'
@@ -422,12 +453,24 @@ jobs:
         with:
           ref: gh-pages-beta
           path: repo/
+      - name: Download default test collection results
+        # Default test results in case the integration tests time out or runner set up fails
+        # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test)
+        uses: actions/download-artifact@v4
+        with:
+          path: allure-collection-default-results/
+          name: allure-collection-default-results-integration-test-charm
       - name: Download test results
         uses: actions/download-artifact@v4
         with:
           path: allure-results/
           pattern: allure-results-integration-test-charm-${{ inputs.cloud }}-juju-${{ inputs.juju-agent-version || needs.integration-test.outputs.juju-snap-channel-for-artifact }}-${{ inputs.architecture }}-*
           merge-multiple: true
+      - name: Combine Allure default results & actual results
+        # For every test: if actual result available, use that. Otherwise, use default result
+        # So that, if actual result not available, Allure report will show "unknown"/"failed" test result
+        # instead of omitting the test
+        run: allure-add-default-for-missing-results --allure-results-dir=allure-results --allure-collection-default-results-dir=allure-collection-default-results
       - name: Load test report history
         run: |
           if [[ -d repo/_latest/history/ ]]

.github/workflows/integration_test_charm_allure_beta.md

@@ -0,0 +1,27 @@
+[Allure Report](https://allurereport.org/) for [integration_test_charm.yaml](integration_test_charm.md)
+
+## Usage
+> [!WARNING]
+> This feature is in beta and **not part of the public interface**. It is subject to breaking changes or removal on a patch version bump.
+
+1. `poetry add --group integration allure-pytest`
+2. Add
+```toml
+allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v0.0.0", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"}
+```
+to your integration test dependencies in `pyproject.toml`.
+
+3. Set `_beta_allure_report: true` for **one** instance of `integration_test_charm.yaml`. If `integration_test_charm.yaml` is called with a matrix, `_beta_allure_report` can only be `true` for one combination of the matrix.
+4. Add permission to `integration_test_charm.yaml` job and all calling workflows
+```yaml
+    permissions:
+      contents: write  # Needed for Allure Report beta
+```
+5. Create gh pages branch
+https://github.com/canonical/data-platform-workflows/blob/5a2c81678ff8733345875235e579d0b1fffbc894/.github/workflows/integration_test_charm.yaml#L363-L371
+6. Enable gh pages publishing at https://github.com/canonical/mysql-router-k8s-operator/settings/pages (replace `mysql-router-k8s-operator` with repository name)
+    ![gh-pages](https://github.com/canonical/data-platform-workflows/assets/115640263/6ee80a1e-f75b-4d67-b11f-977358c32847)
+
+Example for 1, 3-4: https://github.com/canonical/mysql-router-k8s-operator/pull/198
+
+Example for 2:

python/cli/data_platform_workflows_cli/allure_add_default_for_missing_results.py

@@ -0,0 +1,49 @@
+import argparse
+import dataclasses
+import json
+import pathlib
+
+
+@dataclasses.dataclass(frozen=True)
+class Result:
+    test_case_id: str
+    path: pathlib.Path
+
+    def __eq__(self, other):
+        if not isinstance(other, type(self)):
+            return False
+        return self.test_case_id == other.test_case_id
+
+
+def main():
+    """Combine Allure default results & actual results
+
+    For every test: if actual result available, use that. Otherwise, use default result
+
+    So that, if actual result not available, Allure report will show "unknown"/"failed" test result
+    instead of omitting the test
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--allure-results-dir", required=True)
+    parser.add_argument("--allure-collection-default-results-dir", required=True)
+    args = parser.parse_args()
+
+    actual_results = pathlib.Path(args.allure_results_dir)
+    default_results = pathlib.Path(args.allure_collection_default_results_dir)
+
+    results: dict[pathlib.Path, set[Result]] = {
+        actual_results: set(),
+        default_results: set(),
+    }
+    for directory, results_ in results.items():
+        for path in directory.glob("*-result.json"):
+            with path.open("r") as file:
+                id_ = json.load(file)["testCaseId"]
+            results_.add(Result(id_, path))
+
+    actual_results.mkdir(exist_ok=True)
+
+    missing_results = results[default_results] - results[actual_results]
+    for default_result in missing_results:
+        # Move to `actual_results` directory
+        default_result.path.rename(actual_results / default_result.path.name)

python/cli/pyproject.toml

@@ -20,6 +20,7 @@ release-charm = "data_platform_workflows_cli.craft_tools.release:charm"
 update-bundle = "data_platform_workflows_cli.update_bundle:main"
 parse-snap-version = "data_platform_workflows_cli.parse_snap_version:main"
 convert-logsink-to-debug-log = "data_platform_workflows_cli.convert_logsink_to_debug_log:main"
+allure-add-default-for-missing-results = "data_platform_workflows_cli.allure_add_default_for_missing_results:main"
 
 [tool.poetry.dependencies]
 python = "^3.10"

python/pytest_plugins/allure_pytest_collection_report/.gitignore

@@ -0,0 +1 @@
+/poetry.lock

python/pytest_plugins/allure_pytest_collection_report/allure_pytest_collection_report/_plugin.py

@@ -0,0 +1,123 @@
+import allure_commons.logger
+import allure_commons.model2
+import allure_commons.types
+import allure_commons.utils
+import allure_pytest.listener
+import allure_pytest.utils
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--allure-collection-dir",
+        help="Generate default Allure results (used by GitHub Actions) in this directory for tests that are missing Allure results",
+    )
+
+
+def pytest_configure(config):
+    if config.option.allure_collection_dir:
+        config.option.collectonly = True
+
+
+def pytest_collection_finish(session):
+    report_dir = session.config.option.allure_collection_dir
+    if not (report_dir and session.config.option.collectonly):
+        return
+
+    # Copied from `allure_pytest.listener.AllureListener._cache`
+    _cache = allure_pytest.listener.ItemCache()
+    # Modified from `pytest_configure`
+    file_logger = allure_commons.logger.AllureFileLogger(report_dir)
+
+    for item in session.items:
+        # Copied from `allure_pytest.listener.AllureListener.pytest_runtest_protocol`
+        uuid = _cache.push(item.nodeid)
+        test_result = allure_commons.model2.TestResult(
+            name=item.name,
+            uuid=uuid,
+            start=allure_commons.utils.now(),
+            stop=allure_commons.utils.now(),
+        )
+
+        # TODO: remove some metadata?
+
+        # Copied from `allure_pytest.listener.AllureListener.pytest_runtest_setup`
+        params = (
+            allure_pytest.listener.AllureListener._AllureListener__get_pytest_params(
+                item
+            )
+        )
+        test_result.name = allure_pytest.utils.allure_name(item, params)
+        full_name = allure_pytest.utils.allure_full_name(item)
+        test_result.fullName = full_name
+        test_result.testCaseId = allure_commons.utils.md5(full_name)
+        test_result.description = allure_pytest.utils.allure_description(item)
+        test_result.descriptionHtml = allure_pytest.utils.allure_description_html(item)
+        current_param_names = [param.name for param in test_result.parameters]
+        test_result.parameters.extend(
+            [
+                allure_commons.model2.Parameter(
+                    name=name, value=allure_commons.utils.represent(value)
+                )
+                for name, value in params.items()
+                if name not in current_param_names
+            ]
+        )
+
+        # Copied from `allure_pytest.listener.AllureListener.pytest_runtest_teardown`
+        test_result.historyId = allure_pytest.utils.get_history_id(
+            test_result.fullName,
+            test_result.parameters,
+            original_values=allure_pytest.listener.AllureListener._AllureListener__get_pytest_params(
+                item
+            ),
+        )
+        test_result.labels.extend(
+            [
+                allure_commons.model2.Label(name=name, value=value)
+                for name, value in allure_pytest.utils.allure_labels(item)
+            ]
+        )
+        test_result.labels.extend(
+            [
+                allure_commons.model2.Label(
+                    name=allure_commons.types.LabelType.TAG, value=value
+                )
+                for value in allure_pytest.utils.pytest_markers(item)
+            ]
+        )
+        allure_pytest.listener.AllureListener._AllureListener__apply_default_suites(
+            None, item, test_result
+        )
+        test_result.labels.append(
+            allure_commons.model2.Label(
+                name=allure_commons.types.LabelType.HOST,
+                value=allure_commons.utils.host_tag(),
+            )
+        )
+        test_result.labels.append(
+            allure_commons.model2.Label(
+                name=allure_commons.types.LabelType.FRAMEWORK, value="pytest"
+            )
+        )
+        test_result.labels.append(
+            allure_commons.model2.Label(
+                name=allure_commons.types.LabelType.LANGUAGE,
+                value=allure_commons.utils.platform_label(),
+            )
+        )
+        test_result.labels.append(
+            allure_commons.model2.Label(
+                name="package", value=allure_pytest.utils.allure_package(item)
+            )
+        )
+        test_result.links.extend(
+            [
+                allure_commons.model2.Link(link_type, url, name)
+                for link_type, url, name in allure_pytest.utils.allure_links(item)
+            ]
+        )
+
+        # Modified from `allure_pytest.listener.AllureListener.pytest_runtest_protocol`
+        test_result.status = allure_commons.model2.Status.UNKNOWN
+        # Modified from `allure_commons.reporter.AllureReporter.close_test`
+        file_logger.report_result(test_result)

python/pytest_plugins/allure_pytest_collection_report/pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "allure-pytest-collection-report"
+# Version unused; repository has its own versioning system. (See .github/workflows/__release.yaml)
+version = "0.1.0"
+description = ""
+authors = ["Carl Csaposs <[email protected]>"]
+readme = "README.md"
+classifiers = [
+    "Framework :: Pytest",
+]
+
+[tool.poetry.plugins."pytest11"]
+allure_collection_report = "allure_pytest_collection_report._plugin"
+
+[tool.poetry.dependencies]
+python = "^3.8"
+pytest = "*"
+allure-pytest = ">=2.13.5"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"