diff --git a/docs/env-vars.md b/docs/env-vars.md index 781606da..b1f1d2b6 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -40,6 +40,14 @@ | `JMETER_WORKER_CPU_REQUESTS` | Worker CPU requests | | | `JMETER_WORKER_MEMORY_LIMITS` | Worker memory limits | | | `JMETER_WORKER_MEMORY_REQUESTS` | Worker memory requests | | +| `JMETER_WORKER_REMOTE_CUSTOM_DATA_ENABLED` | Enable remote custom data | `false` | +| `JMETER_WORKER_REMOTE_CUSTOM_DATA_BUCKET` | The name of the bucket where remote data is | | +| `JMETER_WORKER_REMOTE_CUSTOM_DATA_VOLUME_SIZE` | Volume size used by download remote data | `1Gi` | +| `RCLONE_CONFIG_REMOTECUSTOMDATA_TYPE` | [Rclone](https://rclone.org/) environment variable for type | | +| `RCLONE_CONFIG_REMOTECUSTOMDATA_ACCESS_KEY_ID` | [Rclone](https://rclone.org/) environment variable for access key ID | | +| `RCLONE_CONFIG_REMOTECUSTOMDATA_SECRET_ACCESS_KEY` | [Rclone](https://rclone.org/) environment variable for secret access key | | +| `RCLONE_CONFIG_REMOTECUSTOMDATA_REGION` | [Rclone](https://rclone.org/) environment variable for region | | +| `RCLONE_CONFIG_REMOTECUSTOMDATA_ENDPOINT` | [Rclone](https://rclone.org/) environment variable for endpoint | | ### Locust | Parameter | Description | Default | diff --git a/docs/jmeter/writing-tests.md b/docs/jmeter/writing-tests.md index 61dafa7f..0a6a1521 100644 --- a/docs/jmeter/writing-tests.md +++ b/docs/jmeter/writing-tests.md @@ -15,6 +15,7 @@ - [Thread group](#thread-group) - [Test with CSV data](#test-with-csv-data) - [Test with environment variables](#test-with-environment-variables) +- [Test with custom data](#test-with-custom-data) ## Introduction @@ -389,3 +390,12 @@ You don't need any special configuration elements to use environment variables i ![http_auth_manager dmg](images/http_auth_manager.png){ height=500 } In the example above the environment variable AUTH_CLIENT_ID used in HTTP Authorization Manager. + +## Test with custom data +Some tests require files as images, JAR files, etc. You can provide this from a S3 Bucket. If the environment variable JMETER_WORKER_REMOTE_CUSTOM_DATA_ENABLED is set to true, before pod creation, a PVC will be created asking the cluster for a volume of size defined in the environment variable JMETER_WORKER_REMOTE_CUSTOM_DATA_VOLUME_SIZE and access mode ReadWriteMany. + +The data will be cloned from the bucket to the volume using [Rclone](https://rclone.org/) and will be available to all the pods. + +For the full list of possible environment variables check [Kangal environment variables](env-vars.md) + +**Attention** The [Dynamic volume provisioning](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/) must be set on the cluster diff --git a/pkg/backends/jmeter/resources.go b/pkg/backends/jmeter/resources.go index a938a5fa..16ea1e85 100644 --- a/pkg/backends/jmeter/resources.go +++ b/pkg/backends/jmeter/resources.go @@ -16,6 +16,7 @@ import ( batchV1 "k8s.io/api/batch/v1" coreV1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -37,6 +38,8 @@ const ( loadTestWorkerServiceName = LoadTestLabel + "-workers" // loadTestWorkerName is the base name of the worker pods loadTestWorkerName = LoadTestLabel + "-worker" + // loadTestWorkerRemoteCustomDataVolumeSize is the default size of custom data volume + loadTestWorkerRemoteCustomDataVolumeSize = "1Gi" // loadTestFile is the name of the config map that is used to hold testfile data loadTestFile = LoadTestLabel + "-testfile" // loadTestMasterJobLabelKey key we are using for the master job label @@ -155,6 +158,31 @@ func (b *Backend) NewTestdataConfigMap(loadTest loadTestV1.LoadTest) ([]*coreV1. return cMaps, nil } +// NewPVC creates a new pvc for customdata +func (b *Backend) NewPVC(loadTest loadTestV1.LoadTest, i int) *coreV1.PersistentVolumeClaim { + volumeSize := loadTestWorkerRemoteCustomDataVolumeSize + if val, ok := loadTest.Spec.EnvVars["JMETER_WORKER_REMOTE_CUSTOM_DATA_VOLUME_SIZE"]; ok { + volumeSize = val + } + return &coreV1.PersistentVolumeClaim{ + ObjectMeta: metaV1.ObjectMeta{ + Name: fmt.Sprintf("pvc-%s", loadTestWorkerName), + Labels: loadTestWorkerPodLabels, + OwnerReferences: []metaV1.OwnerReference{ + *metaV1.NewControllerRef(&loadTest, loadTestV1.SchemeGroupVersion.WithKind("LoadTest")), + }, + }, + Spec: coreV1.PersistentVolumeClaimSpec{ + AccessModes: []coreV1.PersistentVolumeAccessMode{coreV1.ReadWriteMany}, + Resources: coreV1.ResourceRequirements{ + Requests: coreV1.ResourceList{ + coreV1.ResourceName(coreV1.ResourceStorage): resource.MustParse(volumeSize), + }, + }, + }, + } +} + // NewPod creates a new pod which mounts a configmap that contains jmeter testdata func (b *Backend) NewPod(loadTest loadTestV1.LoadTest, i int, configMap *coreV1.ConfigMap, podAnnotations map[string]string) *coreV1.Pod { logger := b.logger.With( @@ -170,7 +198,7 @@ func (b *Backend) NewPod(loadTest loadTestV1.LoadTest, i int, configMap *coreV1. logger.Debug("Loadtest.Spec.WorkerConfig is empty; using worker image from config", zap.String("imageRef", imageRef)) } - return &coreV1.Pod{ + pod := &coreV1.Pod{ ObjectMeta: metaV1.ObjectMeta{ Name: fmt.Sprintf("%s-%03d", loadTestWorkerName, i), Labels: loadTestWorkerPodLabels, @@ -246,6 +274,57 @@ func (b *Backend) NewPod(loadTest loadTestV1.LoadTest, i int, configMap *coreV1. }, }, } + + if _, ok := loadTest.Spec.EnvVars["JMETER_WORKER_REMOTE_CUSTOM_DATA_ENABLED"]; ok { + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, coreV1.VolumeMount{ + Name: "customdata", + MountPath: "/customdata", + }) + pod.Spec.Volumes = append(pod.Spec.Volumes, []coreV1.Volume{ + { + Name: "customdata", + VolumeSource: coreV1.VolumeSource{ + PersistentVolumeClaim: &coreV1.PersistentVolumeClaimVolumeSource{ + ClaimName: fmt.Sprintf("pvc-%s", loadTestWorkerName), + }, + }, + }, + { + Name: "rclone-data", + VolumeSource: coreV1.VolumeSource{ + EmptyDir: &coreV1.EmptyDirVolumeSource{}, + }, + }}...) + pod.Spec.InitContainers = []coreV1.Container{ + { + Name: "get-data", + Image: "rclone/rclone:latest", + Command: []string{"/bin/sh"}, + Args: []string{"-c", "/usr/local/bin/rclone sync remotecustomdata:$(JMETER_WORKER_REMOTE_CUSTOM_DATA_BUCKET) /customdata || echo \"rsync failed\""}, + VolumeMounts: []coreV1.VolumeMount{ + { + Name: "rclone-data", + MountPath: "/data", + }, + { + Name: "customdata", + MountPath: "/customdata", + }, + }, + EnvFrom: []coreV1.EnvFromSource{ + { + SecretRef: &coreV1.SecretEnvSource{ + LocalObjectReference: coreV1.LocalObjectReference{ + Name: loadTestEnvVars, + }, + }, + }, + }, + }, + } + } + + return pod } // NewJMeterMasterJob creates a new job which runs the jmeter master pod @@ -390,6 +469,26 @@ func (b *Backend) CreatePodsWithTestdata(ctx context.Context, configMaps []*core } } + if _, ok := loadTest.Spec.EnvVars["JMETER_WORKER_REMOTE_CUSTOM_DATA_ENABLED"]; ok { + logger.Info("Remote custom data enabled, creating PVC") + + pvc := b.NewPVC(*loadTest, i) + _, err = b.kubeClientSet.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metaV1.CreateOptions{}) + if err != nil && !kerrors.IsAlreadyExists(err) { + logger.Error("Error on creating pvc", zap.Error(err)) + return err + } + + watchObjPvc, err := b.kubeClientSet.CoreV1().PersistentVolumeClaims(namespace).Watch(ctx, metaV1.ListOptions{ + FieldSelector: fmt.Sprintf("metadata.name=%s", pvc.ObjectMeta.Name), + }) + if err != nil { + logger.Warn("unable to watch pvc state", zap.Error(err)) + continue + } + waitfor.Resource(watchObjPvc, (waitfor.Condition{}).PvcReady, b.config.WaitForResourceTimeout) + } + pod := b.NewPod(*loadTest, i, configMap, b.podAnnotations) _, err = b.kubeClientSet.CoreV1().Pods(namespace).Create(ctx, pod, metaV1.CreateOptions{}) if err != nil && !kerrors.IsAlreadyExists(err) { diff --git a/pkg/core/waitfor/waitfor.go b/pkg/core/waitfor/waitfor.go index 56bd9324..5cad7f8b 100644 --- a/pkg/core/waitfor/waitfor.go +++ b/pkg/core/waitfor/waitfor.go @@ -33,6 +33,11 @@ func (Condition) PodRunning(event watch.Event) (bool, error) { return false, nil } +// PvcReady waits until is bound +func (Condition) PvcReady(event watch.Event) (bool, error) { + return coreV1.ClaimBound == event.Object.(*coreV1.PersistentVolumeClaim).Status.Phase, nil +} + // LoadTestRunning waits until Loadtest are with status phase running func (Condition) LoadTestRunning(event watch.Event) (bool, error) { if apisLoadTestV1.LoadTestRunning == event.Object.(*apisLoadTestV1.LoadTest).Status.Phase {