From f3505c03f7c0db222ecae7fb08956cc7fb879488 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Fri, 8 Mar 2024 02:43:11 +0200 Subject: [PATCH 01/12] Remove unused kustomization Signed-off-by: Nir Soffer --- test/basic-test/kustomization.yaml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 test/basic-test/kustomization.yaml diff --git a/test/basic-test/kustomization.yaml b/test/basic-test/kustomization.yaml deleted file mode 100644 index 470ffd416..000000000 --- a/test/basic-test/kustomization.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -# Kustomization template for deploying ramen busybox sample using user -# configurable cluster name. ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ocm-ramen-samples/subscriptions - - ocm-ramen-samples/subscriptions/busybox -patches: - - target: - kind: DRPlacementControl - name: busybox-drpc - patch: |- - - op: replace - path: /spec/preferredCluster - value: $cluster_name From f903731ff34391a8260a0cd898d8d6a59ae27787 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 19:06:29 +0200 Subject: [PATCH 02/12] Lint also enable-dr and disable-dr scripts The script are tiny but it is nice to have verify them with flake8, pylint and black. Signed-off-by: Nir Soffer --- test/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/Makefile b/test/Makefile index 95cb69b56..75ea8d9a4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -15,6 +15,8 @@ sources := $(wildcard \ addons/*/start \ addons/*/test \ addons/*/stop \ + */enable-dr \ + */disable-dr \ */deploy \ */undeploy \ */failover \ From 19f81d5bec47c5c15659f0da6b1d693b0b1267d0 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 13 Nov 2023 21:57:33 +0200 Subject: [PATCH 03/12] Make application path configurable Add "path" config, so test config looks like OpenShift UI: repo: https://github.com/RamenDR/ocm-ramen-samples.git path: subscription branch: main name: busybox-sample namespace: busybox-sample With this we can use basic test to test any subscription based application in ocm-ramen-samples[1] and ocm-kubevirt-samples[2]. [1] https://github.com/RamenDR/ocm-ramen-samples [2] https://github.com/aglitke/ocm-kubevirt-samples Signed-off-by: Nir Soffer --- test/basic-test/config.yaml | 1 + test/drenv/test.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml index ca31c93a7..8413dca4a 100644 --- a/test/basic-test/config.yaml +++ b/test/basic-test/config.yaml @@ -3,6 +3,7 @@ --- repo: https://github.com/ramendr/ocm-ramen-samples.git +path: subscription branch: main name: busybox-sample namespace: busybox-sample diff --git a/test/drenv/test.py b/test/drenv/test.py index 34be0c676..9c28b18cf 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -106,9 +106,9 @@ def deploy(): context=env["hub"], log=debug, ) - info("Deploying subscription based application") + info("Deploying application") kubectl.apply( - f"--kustomize={config['repo']}/subscription?ref={config['branch']}", + f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", context=env["hub"], log=debug, ) @@ -125,9 +125,9 @@ def undeploy(): context=env["hub"], log=debug, ) - info("Undeploying subscription based application") + info("Undeploying application") kubectl.delete( - f"--kustomize={config['repo']}/subscription?ref={config['branch']}", + f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", "--ignore-not-found", context=env["hub"], log=debug, From b074da88823384113be624d86895defb7b780c98 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 13 Nov 2023 23:17:24 +0200 Subject: [PATCH 04/12] Add feature_gates profile option It is possible now to enable Kubernetes feature gates[1] using minikube --feature-gates option[2]. We will use this to enable StatefulSetAutoDeletePVC feature gate. Example config: profiles: - name: featured feature_gates: - StatefulSetAutoDeletePVC=true [1] https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ [2] https://minikube.sigs.k8s.io/docs/handbook/config/#enabling-feature-gates Signed-off-by: Nir Soffer --- test/README.md | 4 ++++ test/drenv/__main__.py | 1 + test/drenv/envfile.py | 1 + test/drenv/minikube.py | 5 +++++ 4 files changed, 11 insertions(+) diff --git a/test/README.md b/test/README.md index 3d2818a9c..70ff2806f 100644 --- a/test/README.md +++ b/test/README.md @@ -466,6 +466,10 @@ $ drenv delete envs/example.yaml - `extra_config`: List of extra config key=value. Each item adds `--extra-config` minikube option. See `minikube start --help` to see the possible keys and values. + - `feature_gates`: List of Kubernetes feature gates key=value. Each + item adds `--feature-gates` minikube option. See + [Feature Gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) + for possible keys and values. - `containerd`: Optional containerd configuration object. See `containerd config default` for available options. - `workers`: Optional list of workers to run when starting a diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index f6cb056ff..a7b181901 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -212,6 +212,7 @@ def start_minikube_cluster(profile, verbose=False): addons=profile["addons"], service_cluster_ip_range=profile["service_cluster_ip_range"], extra_config=profile["extra_config"], + feature_gates=profile["feature_gates"], alsologtostderr=verbose, ) diff --git a/test/drenv/envfile.py b/test/drenv/envfile.py index e132fc599..6e9c39ccc 100644 --- a/test/drenv/envfile.py +++ b/test/drenv/envfile.py @@ -106,6 +106,7 @@ def _validate_profile(profile, addons_root): profile.setdefault("ser", []) profile.setdefault("service_cluster_ip_range", None) profile.setdefault("extra_config", []) + profile.setdefault("feature_gates", []) profile.setdefault("containerd", None) profile.setdefault("workers", []) diff --git a/test/drenv/minikube.py b/test/drenv/minikube.py index 411bab751..ee95ef5bd 100644 --- a/test/drenv/minikube.py +++ b/test/drenv/minikube.py @@ -43,6 +43,7 @@ def start( addons=(), service_cluster_ip_range=None, extra_config=None, + feature_gates=None, alsologtostderr=False, ): args = [] @@ -77,6 +78,10 @@ def start( for pair in extra_config: args.extend(("--extra-config", pair)) + if feature_gates: + # Unlike --extra-config this requires one comma separated value. + args.extend(("--feature-gates", ",".join(feature_gates))) + if alsologtostderr: args.append("--alsologtostderr") From a59264e852f516acdaf75350fb6d0aea95e211a7 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 13 Nov 2023 23:22:17 +0200 Subject: [PATCH 05/12] Enable StatefulSetAutoDeletePVC feature gate With this we can create a statesfulset with persistentVolumeClaimRetentionPolicy[1] to have the PVCs deleted when a stateful set is deleted. This policy is required for relocate, otherwise ramen get stuck waiting for vrs to become secondary. [1] https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention Signed-off-by: Nir Soffer --- test/envs/regional-dr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index 66ea1bfd1..516ae921c 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -21,6 +21,8 @@ templates: addons: - volumesnapshots - csi-hostpath-driver + feature_gates: + - StatefulSetAutoDeletePVC=true workers: - addons: - name: cert-manager From 41576610804814ca50316298d0aab06e9b877c5a Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Thu, 16 Nov 2023 17:45:57 +0200 Subject: [PATCH 06/12] Fix waiting for drcluster and drpolicy We use `--namespace ramen-system` but these are deployed in cluster scope. I guess the `--namespace` is ignored in this case since this code works as is. Signed-off-by: Nir Soffer --- ramenctl/ramenctl/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ramenctl/ramenctl/config.py b/ramenctl/ramenctl/config.py index f3b0b2f91..977d286bb 100644 --- a/ramenctl/ramenctl/config.py +++ b/ramenctl/ramenctl/config.py @@ -145,7 +145,6 @@ def wait_for_dr_clusters(hub, clusters, args): drenv.wait_for( f"drcluster/{name}", output="jsonpath={.status.phase}", - namespace=args.ramen_namespace, timeout=180, profile=hub, log=command.debug, @@ -156,7 +155,6 @@ def wait_for_dr_clusters(hub, clusters, args): "drcluster", "--all", "--for=jsonpath={.status.phase}=Available", - f"--namespace={args.ramen_namespace}", context=hub, log=command.debug, ) @@ -167,7 +165,6 @@ def wait_for_dr_policy(hub, args): kubectl.wait( "drpolicy/dr-policy", "--for=condition=Validated", - f"--namespace={args.ramen_namespace}", context=hub, log=command.debug, ) From 21ace3b9141c3de25d47c0dbb5bbbb93bd3fe9a0 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Thu, 16 Nov 2023 21:35:45 +0200 Subject: [PATCH 07/12] Fix hard coded pvc selector label Add `pvc_label` configuration so we can test any application. With this we can run basic-test with vms from ocm-kubeivrt-samples[1]. [1] https://github.com/aglitke/ocm-kubevirt-samples Signed-off-by: Nir Soffer --- test/basic-test/config.yaml | 1 + test/drenv/test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml index 8413dca4a..ac2081d32 100644 --- a/test/basic-test/config.yaml +++ b/test/basic-test/config.yaml @@ -7,3 +7,4 @@ path: subscription branch: main name: busybox-sample namespace: busybox-sample +pvc_label: busybox diff --git a/test/drenv/test.py b/test/drenv/test.py index 9c28b18cf..b2cf8d334 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -171,7 +171,7 @@ def enable_dr(): name: {placement_name} pvcSelector: matchLabels: - appname: busybox + appname: {config['pvc_label']} """ kubectl.apply("--filename=-", input=drpc, context=env["hub"], log=debug) From e247dacc196bd61553baf47c161c29bc72d1473e Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 19:32:33 +0200 Subject: [PATCH 08/12] Make dr policy configurable Previously the dr policy name was hard coded, which works for drenv default dr policy but not for OpenShift clusters. To run with OpenShift cluster you can either change the dr_policy configuration to match it to the actual dr policy, or create a new policy manually before running the tests. Signed-off-by: Nir Soffer --- test/basic-test/config.yaml | 1 + test/drenv/test.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml index ac2081d32..7937d4d03 100644 --- a/test/basic-test/config.yaml +++ b/test/basic-test/config.yaml @@ -7,4 +7,5 @@ path: subscription branch: main name: busybox-sample namespace: busybox-sample +dr_policy: dr-policy pvc_label: busybox diff --git a/test/drenv/test.py b/test/drenv/test.py index b2cf8d334..29ef9b75d 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -158,14 +158,14 @@ def enable_dr(): apiVersion: ramendr.openshift.io/v1alpha1 kind: DRPlacementControl metadata: - name: busybox-drpc + name: {config['name']}-drpc namespace: {config['namespace']} labels: app: {config['name']} spec: preferredCluster: {cluster} drPolicyRef: - name: dr-policy + name: {config['dr_policy']} placementRef: kind: Placement name: {placement_name} From 69f6885a7d410562a159f19e67a452600d6550c9 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 19:32:51 +0200 Subject: [PATCH 09/12] Move generic logs Since we can test any application now (e.g. busybox, kubvirt), we don't want to mention busybox in the logs. Use config['name'] when we can to make the logs more clear. Signed-off-by: Nir Soffer --- test/basic-test/undeploy | 4 ++-- test/drenv/test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/basic-test/undeploy b/test/basic-test/undeploy index 7fa04aca1..950bf4768 100755 --- a/test/basic-test/undeploy +++ b/test/basic-test/undeploy @@ -8,6 +8,6 @@ from drenv import test test.start("undeploy", __file__) args = test.parse_args() -test.info("Deleting busybox example application") +test.info("Deleting application") test.undeploy() -test.info("Application was undeployed successfully") +test.info("Application was deleted") diff --git a/test/drenv/test.py b/test/drenv/test.py index 29ef9b75d..f3908b433 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -106,7 +106,7 @@ def deploy(): context=env["hub"], log=debug, ) - info("Deploying application") + info("Deploying application '%s'", config["name"]) kubectl.apply( f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", context=env["hub"], @@ -125,7 +125,7 @@ def undeploy(): context=env["hub"], log=debug, ) - info("Undeploying application") + info("Undeploying application '%s'", config["name"]) kubectl.delete( f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", "--ignore-not-found", From 18625e64d7c12b3fb72b1ff2585426c81ab16f6b Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Thu, 7 Mar 2024 21:23:03 +0200 Subject: [PATCH 10/12] Remove channel deployment To run tests concurrently, we cannot delete the channel when we deploy an application, so the test cannot be responsible for deploying and deleting the channel. Creating a channel for the tests is now a manual step documented in docs/testing.md and docs/user-quick-start.md. Signed-off-by: Nir Soffer --- docs/testing.md | 19 +++++++++++++++---- docs/user-quick-start.md | 12 +++++++++--- test/drenv/test.py | 13 ------------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 0fbee1333..2a392f0a7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -66,16 +66,27 @@ The end-to-end testing framework isn't implemented yet. However, we have a basic test that you can use to test the basic flows of Ramen. `basic-test` requires the python virtual environment to be activated. -To run basic tests using the regional-dr environment run: +Ramen basic test use the [ocm-ramen-samples repo](https://github.com/RamenDR/ocm-ramen-samples). +Before running the tests, you need to deploy a channel pointing this +repo: + +```sh +kubectl apply -k https://github.com/RamenDR/ocm-ramen-samples.git/channel?ref=main --context hub +``` + +> [!NOTE] +> To test applications from your repo, you need to deploy a channel +> pointing to your repo. + +To run basic tests using regional-dr environment run: ```sh test/basic-test/run test/envs/regional-dr.yaml ``` -This test: +This test does these operations: -1. Deploys an application using - [ocm-ramen-samples repo](https://github.com/RamenDR/ocm-ramen-samples) +1. Deploys a busybox application 1. Enables DR for the application 1. Fails over the application to the other cluster 1. Relocates the application back to the original cluster diff --git a/docs/user-quick-start.md b/docs/user-quick-start.md index e8bce674b..a485d1b34 100644 --- a/docs/user-quick-start.md +++ b/docs/user-quick-start.md @@ -305,17 +305,23 @@ ramenctl config test/envs/regional-dr.yaml At this point *Ramen* is ready to protect workloads in your cluster, and you are ready for testing basic flows. +Ramen basic test use the [ocm-ramen-samples repo](https://github.com/RamenDR/ocm-ramen-samples). +Before running the tests, you need to deploy a channel pointing this +repo: + +``` +kubectl apply -k https://github.com/RamenDR/ocm-ramen-samples.git/channel --context hub +``` + To run basic tests using regional-dr environment run: ``` -cd ramen test/basic-test/run test/envs/regional-dr.yaml ``` This test does these operations: -1. Deploys an application using - [ocm-ramen-samples repo](https://github.com/RamenDR/ocm-ramen-samples) +1. Deploys a busybox application 1. Enables DR for the application 1. Fails over the application to the other cluster 1. Relocates the application back to the original cluster diff --git a/test/drenv/test.py b/test/drenv/test.py index f3908b433..b62d41590 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -100,12 +100,6 @@ def deploy(): """ Deploy application on cluster. """ - info("Deploying channel") - kubectl.apply( - f"--kustomize={config['repo']}/channel?ref={config['branch']}", - context=env["hub"], - log=debug, - ) info("Deploying application '%s'", config["name"]) kubectl.apply( f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", @@ -118,13 +112,6 @@ def undeploy(): """ Undeploy an application. """ - info("Undeploying channel") - kubectl.delete( - f"--kustomize={config['repo']}/channel?ref={config['branch']}", - "--ignore-not-found", - context=env["hub"], - log=debug, - ) info("Undeploying application '%s'", config["name"]) kubectl.delete( f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", From ba7ce0e6930fbda2161ce7167c447d0972153e61 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 27 Nov 2023 04:11:28 +0200 Subject: [PATCH 11/12] Support multiple configurations basic-test can be run now with custom configuration file. This can be used to run multiple tests concurrently. test/basic-test/run --config rbd-deploy.yaml $env 2>rbd.log & test/basic-test/run --config cephfs-deploy.yaml $env 2>cephfs.log & wait Signed-off-by: Nir Soffer --- test/drenv/test.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/drenv/test.py b/test/drenv/test.py index b62d41590..751f7d014 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -27,8 +27,8 @@ ) -def start(name, file, config_file="config.yaml"): - global workdir, config, parser, log +def start(name, file): + global workdir, parser, log # Setting up logging and sys.excepthook must be first so any failure will # be reported using the logger. @@ -42,10 +42,6 @@ def start(name, file, config_file="config.yaml"): # Working directory for runing the test. workdir = os.path.abspath(os.path.dirname(file)) - config_path = os.path.join(workdir, config_file) - with open(config_path) as f: - config = yaml.safe_load(f) - parser = argparse.ArgumentParser(name) parser.add_argument( "-v", @@ -57,6 +53,12 @@ def start(name, file, config_file="config.yaml"): "--name-prefix", help="Prefix profile names", ) + parser.add_argument( + "-c", + "--config", + default=os.path.join(workdir, "config.yaml"), + help="Test configuration file", + ) parser.add_argument( "filename", help="Environment filename", @@ -68,13 +70,16 @@ def add_argument(*args, **kw): def parse_args(): - global env + global config, env args = parser.parse_args() if args.verbose: log.setLevel(logging.DEBUG) debug("Parsed arguments: %s", args) + with open(args.config) as f: + config = yaml.safe_load(f) + env = ramen.env_info(args.filename, name_prefix=args.name_prefix) debug("Using environment: %s", env) From 0deb4b069637eb4436a000f801bf568616012f0c Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 27 Nov 2023 04:31:29 +0200 Subject: [PATCH 12/12] Add test configurations Add tests configuration for the current subscriptions in ocm-ramen-samples. We will add more configurations when new subscriptions are added. To run test using a custom configuration use: basic-test/run \ --config configs/deployment-k8s-regional-rbd \ env.yaml The default config.yaml is a symlink to the new configuration to keep developers flows working. Developers can create their own test configuration without modifying the source: cp configs/deployment-k8s-regional-rbd.yaml ~/test/myconfig.yaml # edit myconfig.yaml... basic-test/run -c ~/test/myconfig.yaml envs/regional-dr.yaml Signed-off-by: Nir Soffer --- test/basic-test/config.yaml | 12 +----------- test/configs/deployment-k8s-regional-rbd.yaml | 11 +++++++++++ 2 files changed, 12 insertions(+), 11 deletions(-) mode change 100644 => 120000 test/basic-test/config.yaml create mode 100644 test/configs/deployment-k8s-regional-rbd.yaml diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml deleted file mode 100644 index 7937d4d03..000000000 --- a/test/basic-test/config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - ---- -repo: https://github.com/ramendr/ocm-ramen-samples.git -path: subscription -branch: main -name: busybox-sample -namespace: busybox-sample -dr_policy: dr-policy -pvc_label: busybox diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml new file mode 120000 index 000000000..c413e585e --- /dev/null +++ b/test/basic-test/config.yaml @@ -0,0 +1 @@ +../configs/deployment-k8s-regional-rbd.yaml \ No newline at end of file diff --git a/test/configs/deployment-k8s-regional-rbd.yaml b/test/configs/deployment-k8s-regional-rbd.yaml new file mode 100644 index 000000000..1f1aaa5ef --- /dev/null +++ b/test/configs/deployment-k8s-regional-rbd.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/ramendr/ocm-ramen-samples.git +path: subscription/deployment-k8s-regional-rbd +branch: main +name: deployment-rbd +namespace: deployment-rbd +dr_policy: dr-policy +pvc_label: busybox