Support testing any application on any k8s/ocp cluster

docs/testing.md

@@ -66,16 +66,27 @@ The end-to-end testing framework isn't implemented yet. However, we have a basic
 test that you can use to test the basic flows of Ramen. `basic-test` requires
 the python virtual environment to be activated.
 
-To run basic tests using the regional-dr environment run:
+Ramen basic test use the [ocm-ramen-samples repo](https://github.com/RamenDR/ocm-ramen-samples).
+Before running the tests, you need to deploy a channel pointing this
+repo:
+
+```sh
+kubectl apply -k https://github.com/RamenDR/ocm-ramen-samples.git/channel?ref=main --context hub
+```
+
+> [!NOTE]
+> To test applications from your repo, you need to deploy a channel
+> pointing to your repo.
+
+To run basic tests using regional-dr environment run:
 
 ```sh
 test/basic-test/run test/envs/regional-dr.yaml
 ```
 
-This test:
+This test does these operations:
 
-1. Deploys an application using
-   [ocm-ramen-samples repo](https://github.com/RamenDR/ocm-ramen-samples)
+1. Deploys a busybox application
 1. Enables DR for the application
 1. Fails over the application to the other cluster
 1. Relocates the application back to the original cluster

docs/user-quick-start.md

@@ -305,17 +305,23 @@ ramenctl config test/envs/regional-dr.yaml
 At this point *Ramen* is ready to protect workloads in your cluster, and
 you are ready for testing basic flows.
 
+Ramen basic test use the [ocm-ramen-samples repo](https://github.com/RamenDR/ocm-ramen-samples).
+Before running the tests, you need to deploy a channel pointing this
+repo:
+
+```
+kubectl apply -k https://github.com/RamenDR/ocm-ramen-samples.git/channel --context hub
+```
+
 To run basic tests using regional-dr environment run:
 
 ```
-cd ramen
 test/basic-test/run test/envs/regional-dr.yaml
 ```
 
 This test does these operations:
 
-1. Deploys an application using
-   [ocm-ramen-samples repo](https://github.com/RamenDR/ocm-ramen-samples)
+1. Deploys a busybox application
 1. Enables DR for the application
 1. Fails over the application to the other cluster
 1. Relocates the application back to the original cluster

ramenctl/ramenctl/config.py

@@ -145,7 +145,6 @@ def wait_for_dr_clusters(hub, clusters, args):
         drenv.wait_for(
             f"drcluster/{name}",
             output="jsonpath={.status.phase}",
-            namespace=args.ramen_namespace,
             timeout=180,
             profile=hub,
             log=command.debug,
@@ -156,7 +155,6 @@ def wait_for_dr_clusters(hub, clusters, args):
         "drcluster",
         "--all",
         "--for=jsonpath={.status.phase}=Available",
-        f"--namespace={args.ramen_namespace}",
         context=hub,
         log=command.debug,
     )
@@ -167,7 +165,6 @@ def wait_for_dr_policy(hub, args):
     kubectl.wait(
         "drpolicy/dr-policy",
         "--for=condition=Validated",
-        f"--namespace={args.ramen_namespace}",
         context=hub,
         log=command.debug,
     )

test/Makefile

@@ -15,6 +15,8 @@ sources := $(wildcard \
 	addons/*/start \
 	addons/*/test \
 	addons/*/stop \
+	*/enable-dr \
+	*/disable-dr \
 	*/deploy \
 	*/undeploy \
 	*/failover \

test/README.md

@@ -466,6 +466,10 @@ $ drenv delete envs/example.yaml
     - `extra_config`: List of extra config key=value. Each item adds
       `--extra-config` minikube option. See `minikube start --help` to
       see the possible keys and values.
+    - `feature_gates`: List of Kubernetes feature gates key=value. Each
+      item adds `--feature-gates` minikube option. See
+      [Feature Gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/)
+      for possible keys and values.
     - `containerd`: Optional containerd configuration object. See
       `containerd config default` for available options.
     - `workers`: Optional list of workers to run when starting a

test/basic-test/config.yaml

@@ -0,0 +1 @@
+../configs/deployment-k8s-regional-rbd.yaml

test/basic-test/undeploy

@@ -8,6 +8,6 @@ from drenv import test
 test.start("undeploy", __file__)
 args = test.parse_args()
 
-test.info("Deleting busybox example application")
+test.info("Deleting application")
 test.undeploy()
-test.info("Application was undeployed successfully")
+test.info("Application was deleted")

test/configs/deployment-k8s-regional-rbd.yaml

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+---
+repo: https://github.com/ramendr/ocm-ramen-samples.git
+path: subscription/deployment-k8s-regional-rbd
+branch: main
+name: deployment-rbd
+namespace: deployment-rbd
+dr_policy: dr-policy
+pvc_label: busybox

test/drenv/__main__.py

@@ -212,6 +212,7 @@ def start_minikube_cluster(profile, verbose=False):
         addons=profile["addons"],
         service_cluster_ip_range=profile["service_cluster_ip_range"],
         extra_config=profile["extra_config"],
+        feature_gates=profile["feature_gates"],
         alsologtostderr=verbose,
     )
 

test/drenv/envfile.py

@@ -106,6 +106,7 @@ def _validate_profile(profile, addons_root):
     profile.setdefault("ser", [])
     profile.setdefault("service_cluster_ip_range", None)
     profile.setdefault("extra_config", [])
+    profile.setdefault("feature_gates", [])
     profile.setdefault("containerd", None)
     profile.setdefault("workers", [])
 

test/drenv/minikube.py

@@ -43,6 +43,7 @@ def start(
     addons=(),
     service_cluster_ip_range=None,
     extra_config=None,
+    feature_gates=None,
     alsologtostderr=False,
 ):
     args = []
@@ -77,6 +78,10 @@ def start(
         for pair in extra_config:
             args.extend(("--extra-config", pair))
 
+    if feature_gates:
+        # Unlike --extra-config this requires one comma separated value.
+        args.extend(("--feature-gates", ",".join(feature_gates)))
+
     if alsologtostderr:
         args.append("--alsologtostderr")
 

test/drenv/test.py

@@ -27,8 +27,8 @@
 )
 
 
-def start(name, file, config_file="config.yaml"):
-    global workdir, config, parser, log
+def start(name, file):
+    global workdir, parser, log
 
     # Setting up logging and sys.excepthook must be first so any failure will
     # be reported using the logger.
@@ -42,10 +42,6 @@ def start(name, file, config_file="config.yaml"):
     # Working directory for runing the test.
     workdir = os.path.abspath(os.path.dirname(file))
 
-    config_path = os.path.join(workdir, config_file)
-    with open(config_path) as f:
-        config = yaml.safe_load(f)
-
     parser = argparse.ArgumentParser(name)
     parser.add_argument(
         "-v",
@@ -57,6 +53,12 @@ def start(name, file, config_file="config.yaml"):
         "--name-prefix",
         help="Prefix profile names",
     )
+    parser.add_argument(
+        "-c",
+        "--config",
+        default=os.path.join(workdir, "config.yaml"),
+        help="Test configuration file",
+    )
     parser.add_argument(
         "filename",
         help="Environment filename",
@@ -68,13 +70,16 @@ def add_argument(*args, **kw):
 
 
 def parse_args():
-    global env
+    global config, env
 
     args = parser.parse_args()
     if args.verbose:
         log.setLevel(logging.DEBUG)
     debug("Parsed arguments: %s", args)
 
+    with open(args.config) as f:
+        config = yaml.safe_load(f)
+
     env = ramen.env_info(args.filename, name_prefix=args.name_prefix)
     debug("Using environment: %s", env)
 
@@ -100,15 +105,9 @@ def deploy():
     """
     Deploy application on cluster.
     """
-    info("Deploying channel")
+    info("Deploying application '%s'", config["name"])
     kubectl.apply(
-        f"--kustomize={config['repo']}/channel?ref={config['branch']}",
-        context=env["hub"],
-        log=debug,
-    )
-    info("Deploying subscription based application")
-    kubectl.apply(
-        f"--kustomize={config['repo']}/subscription?ref={config['branch']}",
+        f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}",
         context=env["hub"],
         log=debug,
     )
@@ -118,16 +117,9 @@ def undeploy():
     """
     Undeploy an application.
     """
-    info("Undeploying channel")
-    kubectl.delete(
-        f"--kustomize={config['repo']}/channel?ref={config['branch']}",
-        "--ignore-not-found",
-        context=env["hub"],
-        log=debug,
-    )
-    info("Undeploying subscription based application")
+    info("Undeploying application '%s'", config["name"])
     kubectl.delete(
-        f"--kustomize={config['repo']}/subscription?ref={config['branch']}",
+        f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}",
         "--ignore-not-found",
         context=env["hub"],
         log=debug,
@@ -158,20 +150,20 @@ def enable_dr():
 apiVersion: ramendr.openshift.io/v1alpha1
 kind: DRPlacementControl
 metadata:
-  name: busybox-drpc
+  name: {config['name']}-drpc
   namespace: {config['namespace']}
   labels:
     app: {config['name']}
 spec:
   preferredCluster: {cluster}
   drPolicyRef:
-    name: dr-policy
+    name: {config['dr_policy']}
   placementRef:
     kind: Placement
     name: {placement_name}
   pvcSelector:
     matchLabels:
-      appname: busybox
+      appname: {config['pvc_label']}
 """
     kubectl.apply("--filename=-", input=drpc, context=env["hub"], log=debug)
 

test/envs/regional-dr.yaml

@@ -21,6 +21,8 @@ templates:
     addons:
       - volumesnapshots
       - csi-hostpath-driver
+    feature_gates:
+      - StatefulSetAutoDeletePVC=true
     workers:
       - addons:
           - name: cert-manager