Merge 3eb732a into 72ebea6

canonical · Jun 17, 2024 · 6607fed · 6607fed
2 parents 72ebea6 + 3eb732a
commit 6607fed
Show file tree

Hide file tree

Showing 18 changed files with 708 additions and 279 deletions.
diff --git a/.github/workflows/e2e_test_openstack.yaml b/.github/workflows/e2e_test_openstack.yaml
@@ -1,7 +1,8 @@
 name: End-to-End Openstack Test
 
 on:
-  pull_request:
+  #pull_request:
+  workflow_dispatch:
 
 jobs:
   build-charm:
@@ -130,15 +131,15 @@ jobs:
       - name: Copy github-runner Charm
         run: |
           cp  github-runner_ubuntu-22.04-amd64.charm /home/$USER/github-runner_ubuntu-22.04-amd64.charm
-      
+
       - name: Generate Runner Name
         id: runner-name
         run: echo name=${{ matrix.event.abbreviation }}-${{ needs.run-id.outputs.run-id }}${{ github.run_attempt }} >> $GITHUB_OUTPUT
-      
+
       - name: Create Runner OpenStack Flavor
         run: |
           OS_CLIENT_CONFIG_FILE="clouds.yaml" openstack --os-cloud sunbeam flavor create runner --ram 16384 --disk 20 --vcpus 16
-        
+
       - name: Deploy github-runner Charm (Pull Request, Workflow Dispatch and Push)
         if: matrix.event.name == 'workflow_dispatch' || matrix.event.name == 'push' || matrix.event.name == 'pull_request'
         run: |
@@ -153,7 +154,7 @@ jobs:
             --config experimental-openstack-clouds-yaml="$CLOUDS_YAML" \
             --config experimental-openstack-network=demo-network \
             --config experimental-openstack-flavor=runner
-            
+
       - name: Checkout branch (Issues, Schedule)
         if: matrix.event.name == 'issues' || matrix.event.name == 'schedule'
         uses: actions/checkout@v4
@@ -216,7 +217,7 @@ jobs:
             --config experimental-openstack-clouds-yaml="$CLOUDS_YAML" \
             --config experimental-openstack-network=demo-network \
             --config experimental-openstack-flavor=runner
-            
+
       - name: Watch github-runner (Pull Request)
         if: matrix.event.name == 'pull_request'
         env:

diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml
@@ -8,43 +8,42 @@ on:
 jobs:
   # test option values defined at test/conftest.py are passed on via repository secret
   # INTEGRATION_TEST_ARGS to operator-workflows automatically.
-  integration-tests-juju2:
-    name: Integration test with juju 2.9
-    uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
-    secrets: inherit
-    with:
-      juju-channel: 2.9/stable
-      pre-run-script: scripts/pre-integration-test.sh
-      provider: lxd
-      test-tox-env: integration-juju2.9
-      modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]'
-  integration-tests:
-    name: Integration test with juju 3.1
-    uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
-    secrets: inherit
-    with:
-      juju-channel: 3.1/stable
-      pre-run-script: scripts/pre-integration-test.sh
-      provider: lxd
-      test-tox-env: integration-juju3.1
-      modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]'
-  # openstack tests use microstack, whose setup is kind of special
+  # integration-tests-juju2:
+  #   name: Integration test with juju 2.9
+  #   uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
+  #   secrets: inherit
+  #   with:
+  #     juju-channel: 2.9/stable
+  #     pre-run-script: scripts/pre-integration-test.sh
+  #     provider: lxd
+  #     test-tox-env: integration-juju2.9
+  #     modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]'
+  # integration-tests:
+  #   name: Integration test with juju 3.1
+  #   uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
+  #   secrets: inherit
+  #   with:
+  #     juju-channel: 3.1/stable
+  #     pre-run-script: scripts/pre-integration-test.sh
+  #     provider: lxd
+  #     test-tox-env: integration-juju3.1
+  #     modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]'
+  # openstack tests use private-endpoints, whose setup is kind of special
   # - due to the huge resource requirements, we use self-hosted runners for these tests
-  # - microstack requires juju 3.2 and microk8s 1.26
   # - microk8s needs to be preconfigured with the actions operator to work properly with the dockerhub cache
   # - we need to disable the rbac addon for microk8s, otherwise the setup will fail
-  integration-tests-microstack:
-    name: Integration test using microstack
+  integration-tests-private-endpoint:
+    name: Integration test using private-endpoint
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
     secrets: inherit
     with:
       juju-channel: 3.2/stable
-      pre-run-script: scripts/setup-microstack.sh
-      # Setup the microk8s for microstack deployment. The tests runs on LXD.
+      pre-run-script: scripts/setup-lxd.sh
       provider: microk8s
       channel: 1.26-strict/stable
       microk8s-addons: "dns ingress hostpath-storage"
       test-tox-env: integration-juju3.2
       modules: '["test_openstack_runner"]'
+      extra-arguments: "--http-proxy=http://squid.internal:3128 --https-proxy=http://squid.internal:3128 --no-proxy=keystone.ps6.canonical.com,glance.ps6.canonical.com,nova.ps6.canonical.com,neutron.ps6.canonical.com"
       self-hosted-runner: true
-      self-hosted-runner-label: xlarge
+      self-hosted-runner-label: stg-private-endpoint
diff --git a/.github/workflows/manual_test_env.yaml b/.github/workflows/manual_test_env.yaml
@@ -1,24 +1,22 @@
 name: Manual test
 
 on:
-  # TODO: Re-enable
-  # pull_request:
-  workflow_dispatch:
+  pull_request:
+  # workflow_dispatch:
 
 jobs:
-  integration-tests-microstack:
-    name: Integration test using microstack
+  openstack-integration-tests-private-endpoint:
+    name: Integration test using private-endpoint
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
     secrets: inherit
     with:
       juju-channel: 3.2/stable
-      pre-run-script: scripts/setup-microstack.sh
-      provider: microk8s
-      channel: 1.26-strict/stable
-      microk8s-addons: "dns ingress hostpath-storage"
+      pre-run-script: scripts/setup-lxd.sh
+      provider: lxd
       test-tox-env: integration-juju3.2
-      modules: '["test_openstack_runner"]'
+      modules: '["test_e2e",  "test_charm_runner"]'
+      extra-arguments: "-m openstack --openstack-flavor-name=builder-cpu4-ram8-disk50 --http-proxy=http://squid.internal:3128 --https-proxy=http://squid.internal:3128 --no-proxy=keystone.ps6.canonical.com,glance.ps6.canonical.com,nova.ps6.canonical.com,neutron.ps6.canonical.com"
       self-hosted-runner: true
-      self-hosted-runner-label: xlarge
+      self-hosted-runner-label: stg-private-endpoint
       tmate-debug: true
       tmate-timeout: 2400
diff --git a/pyproject.toml b/pyproject.toml
@@ -17,7 +17,6 @@ omit = [
     "src/repo_policy_compliance_client.py",
     # 2024/04/17: These files are pending a major refactor. The refactor includes a RunnerManager
     # interface class which will include a complete re-organization of the code in these files.
-    "src/openstack_cloud/openstack_manager.py",
     "src/runner.py",
     "src/runner_manager.py",
 ]

diff --git a/scripts/setup-lxd.sh b/scripts/setup-lxd.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+#  Copyright 2024 Canonical Ltd.
+#  See LICENSE file for licensing details.
+
+# Script to setup LXD for testing
+
+juju bootstrap localhost lxd
diff --git a/src-docs/openstack_manager.md b/src-docs/openstack_manager.md
@@ -13,11 +13,14 @@ Module for handling interactions with OpenStack.
 - **SECURITY_GROUP_NAME**
 - **BUILD_OPENSTACK_IMAGE_SCRIPT_FILENAME**
 - **MAX_METRICS_FILE_SIZE**
+- **RUNNER_STARTUP_PROCESS**
+- **RUNNER_LISTENER_PROCESS**
+- **RUNNER_WORKER_PROCESS**
 - **CREATE_SERVER_TIMEOUT**
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L347"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L350"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>function</kbd> `build_image`
 
@@ -57,7 +60,7 @@ Build and upload an image to OpenStack.
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L406"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L409"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>function</kbd> `create_instance_config`
 
@@ -93,7 +96,7 @@ Create an instance config from charm data.
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L120"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L123"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `InstanceConfig`
 The configuration values for creating a single runner instance. 
@@ -132,7 +135,7 @@ __init__(
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L195"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L198"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `ProxyStringValues`
 Wrapper class to proxy values to string. 
@@ -151,7 +154,7 @@ Wrapper class to proxy values to string.
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L303"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L306"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `OpenstackUpdateImageError`
 Represents an error while updating image on Openstack. 
@@ -162,7 +165,7 @@ Represents an error while updating image on Openstack.
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L500"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L503"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `GithubRunnerRemoveError`
 Represents an error removing registered runner from Github. 
@@ -173,7 +176,7 @@ Represents an error removing registered runner from Github.
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L508"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L513"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `OpenstackRunnerManager`
 Runner manager for OpenStack-based instances. 
@@ -186,7 +189,7 @@ Runner manager for OpenStack-based instances.
  - <b>`unit_num`</b>:  The juju unit number. 
  - <b>`instance_name`</b>:  Prefix of the name for the set of runners. 
 
-<a href="../src/openstack_cloud/openstack_manager.py#L517"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L522"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ### <kbd>method</kbd> `__init__`
 
@@ -215,7 +218,7 @@ Construct OpenstackRunnerManager object.
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L1388"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L1713"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ### <kbd>method</kbd> `flush`
 
@@ -232,7 +235,7 @@ Flush Openstack servers.
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L876"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L605"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ### <kbd>method</kbd> `get_github_runner_info`
 
@@ -249,7 +252,7 @@ Get information on GitHub for the runners.
 
 ---
 
-<a href="../src/openstack_cloud/openstack_manager.py#L1218"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/openstack_cloud/openstack_manager.py#L553"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ### <kbd>method</kbd> `reconcile`
 

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -52,11 +52,6 @@ def pytest_addoption(parser: Parser):
         action="store",
         help="The loop device to create shared FS for metrics logging",
     )
-    parser.addoption(
-        "--openstack-clouds-yaml",
-        action="store",
-        help="The OpenStack clouds yaml file for the charm to use.",
-    )
     parser.addoption(
         "--use-existing-app",
         action="store",
@@ -66,3 +61,66 @@ def pytest_addoption(parser: Parser):
         "It is expected that the existing app is already integrated with other apps "
         "like grafana-agent, etc. ",
     )
+    # Openstack testing opts
+    parser.addoption(
+        "--openstack-network-name",
+        action="store",
+        help="The Openstack network to create testing instances under.",
+        default=None,
+    )
+    parser.addoption(
+        "--openstack-flavor-name",
+        action="store",
+        help="The Openstack flavor to create testing instances with.",
+        default=None,
+    )
+    # microstack local testing option
+    parser.addoption(
+        "--openstack-clouds-yaml",
+        action="store",
+        help="The OpenStack clouds yaml file for the charm to use.",
+        default=None,
+    )
+    # Private endpoint options
+    parser.addoption(
+        "--openstack-auth-url",
+        action="store",
+        help="The URL to Openstack authentication service, i.e. keystone.",
+        default=None,
+    )
+    parser.addoption(
+        "--openstack-password",
+        action="store",
+        help="The password to authenticate to Openstack service.",
+        default=None,
+    )
+    parser.addoption(
+        "--openstack-project-domain-name",
+        action="store",
+        help="The Openstack project domain name to use.",
+        default=None,
+    )
+    parser.addoption(
+        "--openstack-project-name",
+        action="store",
+        help="The Openstack project name to use.",
+        default=None,
+    )
+    parser.addoption(
+        "--openstack-user-domain-name",
+        action="store",
+        help="The Openstack user domain name to use.",
+        default=None,
+    )
+    parser.addoption(
+        "--openstack-username",
+        action="store",
+        help="The Openstack user to authenticate as.",
+        default=None,
+    )
+    parser.addoption(
+        "--openstack-region-name",
+        action="store",
+        help="The Openstack region to authenticate to.",
+        default=None,
+    )